From 50a450c8a125fc8bb3a9fd986d3003f8ec2a2e0d Mon Sep 17 00:00:00 2001 From: Devananda van der Veen Date: Sun, 5 May 2013 06:52:08 -0700 Subject: [PATCH] un-split the db backend --- ironic/db/sqlalchemy/api.py | 111 ++++++++++++++++-------------- ironic/db/sqlalchemy/migration.py | 28 ++++---- ironic/db/sqlalchemy/models.py | 17 +++-- ironic/db/sqlalchemy/session.py | 65 ----------------- ironic/tests/db/base.py | 8 +-- 5 files changed, 90 insertions(+), 139 deletions(-) delete mode 100644 ironic/db/sqlalchemy/session.py diff --git a/ironic/db/sqlalchemy/api.py b/ironic/db/sqlalchemy/api.py index ff529ae1f1..31db274a84 100644 --- a/ironic/db/sqlalchemy/api.py +++ b/ironic/db/sqlalchemy/api.py @@ -25,55 +25,91 @@ import uuid from sqlalchemy.sql.expression import asc from sqlalchemy.sql.expression import literal_column -import nova.context -from nova.db.sqlalchemy import api as sqlalchemy_api -from nova import exception +import ironic.context +from ironic import exception from ironic.openstack.common.db import exception as db_exc +from ironic.openstack.common.db.sqlalchemy import session as db_session +from ironic.openstack.common import log as logging from ironic.openstack.common import timeutils from ironic.openstack.common import uuidutils -from nova.virt.baremetal.db.sqlalchemy import models -from nova.virt.baremetal.db.sqlalchemy import session as db_session +from ironic.db.sqlalchemy import models -def model_query(context, *args, **kwargs): +LOG = logging.getLogger(__name__) + +get_engine = db_session.get_engine +get_session = db_session.get_session + +def _retry_on_deadlock(f): + """Decorator to retry a DB API call if Deadlock was received.""" + @functools.wraps(f) + def wrapped(*args, **kwargs): + while True: + try: + return f(*args, **kwargs) + except db_exc.DBDeadlock: + LOG.warn(_("Deadlock detected when running " + "'%(func_name)s': Retrying..."), + dict(func_name=f.__name__)) + # Retry! + time.sleep(0.5) + continue + functools.update_wrapper(wrapped, f) + return wrapped + + +def model_query(context, model, *args, **kwargs): """Query helper that accounts for context's `read_deleted` field. :param context: context to query under :param session: if present, the session to use :param read_deleted: if present, overrides context's read_deleted field. :param project_only: if present and context is user-type, then restrict - query to match the context's project_id. + query to match the context's project_id. If set to 'allow_none', + restriction includes project_id = None. + :param base_model: Where model_query is passed a "model" parameter which is + not a subclass of NovaBase, we should pass an extra base_model + parameter that is a subclass of NovaBase and corresponds to the + model parameter. """ - session = kwargs.get('session') or db_session.get_session() + session = kwargs.get('session') or get_session() read_deleted = kwargs.get('read_deleted') or context.read_deleted - project_only = kwargs.get('project_only') + project_only = kwargs.get('project_only', False) - query = session.query(*args) + def issubclassof_nova_base(obj): + return isinstance(obj, type) and issubclass(obj, models.NovaBase) + base_model = model + if not issubclassof_nova_base(base_model): + base_model = kwargs.get('base_model', None) + if not issubclassof_nova_base(base_model): + raise Exception(_("model or base_model parameter should be " + "subclass of NovaBase")) + + query = session.query(model, *args) + + default_deleted_value = base_model.__mapper__.c.deleted.default.arg if read_deleted == 'no': - query = query.filter_by(deleted=False) + query = query.filter(base_model.deleted == default_deleted_value) elif read_deleted == 'yes': pass # omit the filter to include deleted and active elif read_deleted == 'only': - query = query.filter_by(deleted=True) + query = query.filter(base_model.deleted != default_deleted_value) else: - raise Exception( - _("Unrecognized read_deleted value '%s'") % read_deleted) + raise Exception(_("Unrecognized read_deleted value '%s'") + % read_deleted) - if project_only and nova.context.is_user_context(context): - query = query.filter_by(project_id=context.project_id) + if nova.context.is_user_context(context) and project_only: + if project_only == 'allow_none': + query = query.\ + filter(or_(base_model.project_id == context.project_id, + base_model.project_id == None)) + else: + query = query.filter_by(project_id=context.project_id) return query -def _save(ref, session=None): - if not session: - session = db_session.get_session() - # We must not call ref.save() with session=None, otherwise NovaBase - # uses nova-db's session, which cannot access bm-db. - ref.save(session=session) - - def _build_node_order_by(query): query = query.order_by(asc(models.BareMetalNode.memory_mb)) query = query.order_by(asc(models.BareMetalNode.cpus)) @@ -81,7 +117,6 @@ def _build_node_order_by(query): return query -@sqlalchemy_api.require_admin_context def bm_node_get_all(context, service_host=None): query = model_query(context, models.BareMetalNode, read_deleted="no") if service_host: @@ -89,7 +124,6 @@ def bm_node_get_all(context, service_host=None): return query.all() -@sqlalchemy_api.require_admin_context def bm_node_get_associated(context, service_host=None): query = model_query(context, models.BareMetalNode, read_deleted="no").\ filter(models.BareMetalNode.instance_uuid is not None) @@ -98,7 +132,6 @@ def bm_node_get_associated(context, service_host=None): return query.all() -@sqlalchemy_api.require_admin_context def bm_node_get_unassociated(context, service_host=None): query = model_query(context, models.BareMetalNode, read_deleted="no").\ filter(models.BareMetalNode.instance_uuid is None) @@ -107,7 +140,6 @@ def bm_node_get_unassociated(context, service_host=None): return query.all() -@sqlalchemy_api.require_admin_context def bm_node_find_free(context, service_host=None, cpus=None, memory_mb=None, local_gb=None): query = model_query(context, models.BareMetalNode, read_deleted="no") @@ -124,7 +156,6 @@ def bm_node_find_free(context, service_host=None, return query.first() -@sqlalchemy_api.require_admin_context def bm_node_get(context, bm_node_id): # bm_node_id may be passed as a string. Convert to INT to improve DB perf. bm_node_id = int(bm_node_id) @@ -138,7 +169,6 @@ def bm_node_get(context, bm_node_id): return result -@sqlalchemy_api.require_admin_context def bm_node_get_by_instance_uuid(context, instance_uuid): if not uuidutils.is_uuid_like(instance_uuid): raise exception.InstanceNotFound(instance_id=instance_uuid) @@ -153,7 +183,6 @@ def bm_node_get_by_instance_uuid(context, instance_uuid): return result -@sqlalchemy_api.require_admin_context def bm_node_get_by_node_uuid(context, bm_node_uuid): result = model_query(context, models.BareMetalNode, read_deleted="no").\ filter_by(uuid=bm_node_uuid).\ @@ -165,7 +194,6 @@ def bm_node_get_by_node_uuid(context, bm_node_uuid): return result -@sqlalchemy_api.require_admin_context def bm_node_create(context, values): if not values.get('uuid'): values['uuid'] = str(uuid.uuid4()) @@ -175,7 +203,6 @@ def bm_node_create(context, values): return bm_node_ref -@sqlalchemy_api.require_admin_context def bm_node_update(context, bm_node_id, values): rows = model_query(context, models.BareMetalNode, read_deleted="no").\ filter_by(id=bm_node_id).\ @@ -185,7 +212,6 @@ def bm_node_update(context, bm_node_id, values): raise exception.NodeNotFound(node_id=bm_node_id) -@sqlalchemy_api.require_admin_context def bm_node_associate_and_update(context, node_uuid, values): """Associate an instance to a node safely @@ -216,7 +242,6 @@ def bm_node_associate_and_update(context, node_uuid, values): return ref -@sqlalchemy_api.require_admin_context def bm_node_destroy(context, bm_node_id): # First, delete all interfaces belonging to the node. # Delete physically since these have unique columns. @@ -235,13 +260,11 @@ def bm_node_destroy(context, bm_node_id): raise exception.NodeNotFound(node_id=bm_node_id) -@sqlalchemy_api.require_admin_context def bm_pxe_ip_get_all(context): query = model_query(context, models.BareMetalPxeIp, read_deleted="no") return query.all() -@sqlalchemy_api.require_admin_context def bm_pxe_ip_create(context, address, server_address): ref = models.BareMetalPxeIp() ref.address = address @@ -250,7 +273,6 @@ def bm_pxe_ip_create(context, address, server_address): return ref -@sqlalchemy_api.require_admin_context def bm_pxe_ip_create_direct(context, bm_pxe_ip): ref = bm_pxe_ip_create(context, address=bm_pxe_ip['address'], @@ -258,7 +280,6 @@ def bm_pxe_ip_create_direct(context, bm_pxe_ip): return ref -@sqlalchemy_api.require_admin_context def bm_pxe_ip_destroy(context, ip_id): # Delete physically since it has unique columns model_query(context, models.BareMetalPxeIp, read_deleted="no").\ @@ -266,7 +287,6 @@ def bm_pxe_ip_destroy(context, ip_id): delete() -@sqlalchemy_api.require_admin_context def bm_pxe_ip_destroy_by_address(context, address): # Delete physically since it has unique columns model_query(context, models.BareMetalPxeIp, read_deleted="no").\ @@ -274,7 +294,6 @@ def bm_pxe_ip_destroy_by_address(context, address): delete() -@sqlalchemy_api.require_admin_context def bm_pxe_ip_get(context, ip_id): result = model_query(context, models.BareMetalPxeIp, read_deleted="no").\ filter_by(id=ip_id).\ @@ -283,7 +302,6 @@ def bm_pxe_ip_get(context, ip_id): return result -@sqlalchemy_api.require_admin_context def bm_pxe_ip_get_by_bm_node_id(context, bm_node_id): result = model_query(context, models.BareMetalPxeIp, read_deleted="no").\ filter_by(bm_node_id=bm_node_id).\ @@ -295,7 +313,6 @@ def bm_pxe_ip_get_by_bm_node_id(context, bm_node_id): return result -@sqlalchemy_api.require_admin_context def bm_pxe_ip_associate(context, bm_node_id): session = db_session.get_session() with session.begin(): @@ -333,14 +350,12 @@ def bm_pxe_ip_associate(context, bm_node_id): return ip_ref.id -@sqlalchemy_api.require_admin_context def bm_pxe_ip_disassociate(context, bm_node_id): model_query(context, models.BareMetalPxeIp, read_deleted="no").\ filter_by(bm_node_id=bm_node_id).\ update({'bm_node_id': None}) -@sqlalchemy_api.require_admin_context def bm_interface_get(context, if_id): result = model_query(context, models.BareMetalInterface, read_deleted="no").\ @@ -354,14 +369,12 @@ def bm_interface_get(context, if_id): return result -@sqlalchemy_api.require_admin_context def bm_interface_get_all(context): query = model_query(context, models.BareMetalInterface, read_deleted="no") return query.all() -@sqlalchemy_api.require_admin_context def bm_interface_destroy(context, if_id): # Delete physically since it has unique columns model_query(context, models.BareMetalInterface, read_deleted="no").\ @@ -369,7 +382,6 @@ def bm_interface_destroy(context, if_id): delete() -@sqlalchemy_api.require_admin_context def bm_interface_create(context, bm_node_id, address, datapath_id, port_no): ref = models.BareMetalInterface() ref.bm_node_id = bm_node_id @@ -380,7 +392,6 @@ def bm_interface_create(context, bm_node_id, address, datapath_id, port_no): return ref.id -@sqlalchemy_api.require_admin_context def bm_interface_set_vif_uuid(context, if_id, vif_uuid): session = db_session.get_session() with session.begin(): @@ -406,7 +417,6 @@ def bm_interface_set_vif_uuid(context, if_id, vif_uuid): raise e -@sqlalchemy_api.require_admin_context def bm_interface_get_by_vif_uuid(context, vif_uuid): result = model_query(context, models.BareMetalInterface, read_deleted="no").\ @@ -420,7 +430,6 @@ def bm_interface_get_by_vif_uuid(context, vif_uuid): return result -@sqlalchemy_api.require_admin_context def bm_interface_get_all_by_bm_node_id(context, bm_node_id): result = model_query(context, models.BareMetalInterface, read_deleted="no").\ diff --git a/ironic/db/sqlalchemy/migration.py b/ironic/db/sqlalchemy/migration.py index 7af6bc7352..4425a22aff 100644 --- a/ironic/db/sqlalchemy/migration.py +++ b/ironic/db/sqlalchemy/migration.py @@ -17,15 +17,17 @@ # under the License. import distutils.version as dist_version +import os + +from ironic.db import migration +from ironic import exception +from ironic.openstack.common.db.sqlalchemy import session as db_session + + import migrate from migrate.versioning import util as migrate_util -import os import sqlalchemy -from ironic import exception -from ironic.db import migration -from ironic.db.sqlalchemy import session - @migrate_util.decorator def patched_with_engine(f, *a, **kw): @@ -54,9 +56,10 @@ from migrate import exceptions as versioning_exceptions from migrate.versioning import api as versioning_api from migrate.versioning.repository import Repository - _REPOSITORY = None +get_engine = db_session.get_engine + def db_sync(version=None): if version is not None: @@ -68,25 +71,24 @@ def db_sync(version=None): current_version = db_version() repository = _find_migrate_repo() if version is None or version > current_version: - return versioning_api.upgrade(session.get_engine(), repository, - version) + return versioning_api.upgrade(get_engine(), repository, version) else: - return versioning_api.downgrade(session.get_engine(), repository, + return versioning_api.downgrade(get_engine(), repository, version) def db_version(): repository = _find_migrate_repo() try: - return versioning_api.db_version(session.get_engine(), repository) + return versioning_api.db_version(get_engine(), repository) except versioning_exceptions.DatabaseNotControlledError: meta = sqlalchemy.MetaData() - engine = session.get_engine() + engine = get_engine() meta.reflect(bind=engine) tables = meta.tables if len(tables) == 0: db_version_control(migration.INIT_VERSION) - return versioning_api.db_version(session.get_engine(), repository) + return versioning_api.db_version(get_engine(), repository) else: # Some pre-Essex DB's may not be version controlled. # Require them to upgrade using Essex first. @@ -96,7 +98,7 @@ def db_version(): def db_version_control(version=None): repository = _find_migrate_repo() - versioning_api.version_control(session.get_engine(), repository, version) + versioning_api.version_control(get_engine(), repository, version) return version diff --git a/ironic/db/sqlalchemy/models.py b/ironic/db/sqlalchemy/models.py index 61063f0311..a330558ea7 100644 --- a/ironic/db/sqlalchemy/models.py +++ b/ironic/db/sqlalchemy/models.py @@ -21,15 +21,20 @@ SQLAlchemy models for baremetal data. from sqlalchemy import Column, Boolean, Integer, String from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy import ForeignKey, Text - -from nova.db.sqlalchemy import models +from sqlalchemy import ForeignKey, Text, DateTime, Float +from ironic.openstack.common.db.sqlalchemy import models BASE = declarative_base() -class BareMetalNode(BASE, models.NovaBase): +class IronicBase(models.SoftDeleteMixin, + models.TimestampMixin, + models.ModelBase): + metadata = None + + +class BareMetalNode(BASE, IronicBase): """Represents a bare metal node.""" __tablename__ = 'bm_nodes' @@ -55,7 +60,7 @@ class BareMetalNode(BASE, models.NovaBase): swap_mb = Column(Integer) -class BareMetalPxeIp(BASE, models.NovaBase): +class BareMetalPxeIp(BASE, IronicBase): __tablename__ = 'bm_pxe_ips' id = Column(Integer, primary_key=True) deleted = Column(Boolean, default=False) @@ -64,7 +69,7 @@ class BareMetalPxeIp(BASE, models.NovaBase): bm_node_id = Column(Integer, ForeignKey('bm_nodes.id'), nullable=True) -class BareMetalInterface(BASE, models.NovaBase): +class BareMetalInterface(BASE, IronicBase): __tablename__ = 'bm_interfaces' id = Column(Integer, primary_key=True) deleted = Column(Boolean, default=False) diff --git a/ironic/db/sqlalchemy/session.py b/ironic/db/sqlalchemy/session.py deleted file mode 100644 index c23bb317d8..0000000000 --- a/ironic/db/sqlalchemy/session.py +++ /dev/null @@ -1,65 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2012 NTT DOCOMO, INC. -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Session Handling for SQLAlchemy backend.""" - -from oslo.config import cfg - -from ironic.openstack.common.db.sqlalchemy import session as nova_session -from nova import paths - -opts = [ - cfg.StrOpt('sql_connection', - default=('sqlite:///' + - paths.state_path_def('baremetal_$sqlite_db')), - help='The SQLAlchemy connection string used to connect to the ' - 'bare-metal database'), - ] - -baremetal_group = cfg.OptGroup(name='baremetal', - title='Baremetal Options') - -CONF = cfg.CONF -CONF.register_group(baremetal_group) -CONF.register_opts(opts, baremetal_group) - -CONF.import_opt('sqlite_db', 'ironic.openstack.common.db.sqlalchemy.session') - -_ENGINE = None -_MAKER = None - - -def get_session(autocommit=True, expire_on_commit=False): - """Return a SQLAlchemy session.""" - global _MAKER - - if _MAKER is None: - engine = get_engine() - _MAKER = nova_session.get_maker(engine, autocommit, expire_on_commit) - - session = _MAKER() - return session - - -def get_engine(): - """Return a SQLAlchemy engine.""" - global _ENGINE - if _ENGINE is None: - _ENGINE = nova_session.create_engine(CONF.baremetal.sql_connection) - return _ENGINE diff --git a/ironic/tests/db/base.py b/ironic/tests/db/base.py index 7ece63c9fd..3a026ceeb3 100644 --- a/ironic/tests/db/base.py +++ b/ironic/tests/db/base.py @@ -19,14 +19,14 @@ from oslo.config import cfg from ironic import context as ironic_context from ironic import test -from ironic.db import migration as bm_migration -from ironic.db.sqlalchemy import session as bm_session +from ironic.db import migration as db_migration +from ironic.openstack.common.db.sqlalchemy import session as db_session _DB_CACHE = None CONF = cfg.CONF CONF.import_opt('sql_connection', - 'ironic.db.sqlalchemy.session') + 'ironic.openstack.common.db.sqlalchemy.session') class Database(test.Database): @@ -42,7 +42,7 @@ class BMDBTestCase(test.TestCase): self.flags(sql_connection='sqlite://') global _DB_CACHE if not _DB_CACHE: - _DB_CACHE = Database(bm_session, bm_migration, + _DB_CACHE = Database(db_session, db_migration, sql_connection=CONF.sql_connection, sqlite_db=None, sqlite_clean_db=None)