Adding the sqlalchemy db implmentation

etc/heat-engine.conf

@@ -25,3 +25,5 @@ use_syslog = False
 # syslog_log_facility = LOG_LOCAL0
 
 sql_connection = mysql://heat:heat@localhost/heat
+
+db_backend=heat.db.sqlalchemy.api

heat/common/config.py

@@ -51,6 +51,13 @@ class HeatEngineConfigOpts(HeatConfigOpts):
 
     db_opts = [
     cfg.StrOpt('db_backend', default='heat.db.anydbm.api', help='The backend to use for db'),
+    cfg.StrOpt('sql_connection',
+               default='mysql://heat:heat@localhost/heat',
+               help='The SQLAlchemy connection string used to connect to the '
+                    'database'),
+    cfg.IntOpt('sql_idle_timeout',
+               default=3600,
+               help='timeout before idle sql connections are reaped'),
     ]
 
     def __init__(self, **kwargs):

heat/db/api.py

@@ -26,12 +26,15 @@ The underlying driver is loaded as a :class:`LazyPluggable`. SQLAlchemy is
 currently the only supported backend.
 '''
 
-from heat.openstack.common import cfg
 from heat.common import utils
 
 def configure(conf):
     global IMPL
+    global SQL_CONNECTION
+    global SQL_IDLE_TIMEOUT
     IMPL = utils.import_object(conf.db_backend)
+    SQL_CONNECTION = conf.sql_connection
+    SQL_IDLE_TIMEOUT = conf.sql_idle_timeout
 
 def raw_template_get(context, template_id):
     return IMPL.raw_template_get(context, template_id)
@@ -53,14 +56,24 @@ def parsed_template_create(context, values):
     return IMPL.parsed_template_create(context, values)
 
 
-def state_get(context, state_id):
-    return IMPL.state_get(context, state_id)
+def resource_get(context, resource_id):
+    return IMPL.resource_get(context, resource_id)
 
-def state_get_all(context):
-    return IMPL.state_get_all(context)
+def resource_get_all(context):
+    return IMPL.resource_get_all(context)
 
-def state_create(context, values):
-    return IMPL.state_create(context, values)
+def resource_create(context, values):
+    return IMPL.resource_create(context, values)
+
+
+def stack_get(context, stack_id):
+    return IMPL.resource_get(context, resource_id)
+
+def stack_get_all(context):
+    return IMPL.stack_get_all(context)
+
+def stack_create(context, values):
+    return IMPL.stack_create(context, values)
 
 
 def event_get(context, event_id):

heat/db/sqlalchemy/api.py

@@ -15,11 +15,9 @@
 
 '''Implementation of SQLAlchemy backend.'''
 
-from nova.db.sqlalchemy.session import get_session
-from nova import flags
 from nova import utils
-
-FLAGS = flags.FLAGS
+from heat.db.sqlalchemy import models
+from heat.db.sqlalchemy.session import get_session
 
 def model_query(context, *args, **kwargs):
     """Query helper that accounts for context's `read_deleted` field.
@@ -51,45 +49,126 @@ def model_query(context, *args, **kwargs):
 
     return query
 
-# a big TODO
 def raw_template_get(context, template_id):
-    return 'test return value'
+    result = model_query(context, models.RawTemplate).\
+                        filter_by(id=template_id).first()
+
+    if not result:
+        raise Exception("raw template with id %s not found" % template_id)
+
+    return result
 
 def raw_template_get_all(context):
-    pass
+    results = model_query(context, models.RawTemplate).all()
+
+    if not results:
+        raise Exception('no raw templates were found')
+    
+    return results
 
 def raw_template_create(context, values):
-    pass
-
+    raw_template_ref = models.RawTemplate()
+    raw_template_ref.update(values)
+    raw_template_ref.save()
+    return raw_template_ref
 
 def parsed_template_get(context, template_id):
-    pass
+    result = model_query(context, models.ParsedTemplate).\
+                        filter_by(id=template_id).first()
+
+    if not result:
+        raise Exception("parsed template with id %s not found" % template_id)
+
+    return result
 
 def parsed_template_get_all(context):
-    pass
+    results = model_query(context, models.ParsedTemplate).all()
+
+    if not results:
+        raise Exception('no parsed templates were found')
+    
+    return results
 
 def parsed_template_create(context, values):
-    pass
+    parsed_template_ref = models.ParsedTemplate()
+    parsed_template_ref.update(values)
+    parsed_template_ref.save()
+    return parsed_template_ref
 
+def resource_get(context, resource_id):
+    result = model_query(context, models.Resource).\
+                        filter_by(id=resource_id).first()
 
-def state_get(context, state_id):
-    pass
+    if not result:
+        raise Exception("resource with id %s not found" % resource_id)
 
-def state_get_all(context):
-    pass
+    return result
 
-def state_create(context, values):
-    pass
+def resource_get_all(context):
+    results = model_query(context, models.Resource).all()
 
+    if not results:
+        raise Exception('no resources were found')
+    
+    return results
+
+def resource_create(context, values):
+    resource_ref = models.Resource()
+    resource_ref.update(values)
+    resource_ref.save()
+    return resource_ref
+
+def stack_get(context, stack_id):
+    result = model_query(context, models.Stack).\
+                        filter_by(id=stack_id).first()
+
+    if not result:
+        raise Exception("stack with id %s not found" % stack_id)
+
+    return result
+
+def stack_get_all(context):
+    results = model_query(context, models.Stack).all()
+
+    if not results:
+        raise Exception('no stacks were found')
+    
+    return results
+
+def stack_create(context, values):
+    stack_ref = models.Stack()
+    stack_ref.update(values)
+    stack_ref.save()
+    return stack_ref
 
 def event_get(context, event_id):
-    pass
+    result = model_query(context, models.Event).\
+                        filter_by(id=event_id).first()
+
+    if not result:
+        raise Exception("event with id %s not found" % event_id)
+
+    return result
 
 def event_get_all(context):
-    pass
+    results = model_query(context, models.Event).all()
+
+    if not results:
+        raise Exception('no events were found')
+    
+    return results
 
 def event_get_all_by_stack(context, stack_id):
-    pass
+    results = model_query(context, models.Event).\
+                        filter_by(stack_id).all()
+
+    if not results:
+        raise Exception("no events for stack_id %s were found" % stack_id)
+    
+    return results
 
 def event_create(context, values):
-    pass
+    event_ref = models.Event()
+    event_ref.update(values)
+    event_ref.save()
+    return event_ref

heat/db/sqlalchemy/migrate_repo/versions/001_norwhal.py

@@ -13,9 +13,20 @@ def upgrade(migrate_engine):
         Column('template', Text()),
     )
 
+    stack = Table(
+        'stack', meta,
+        Column('id', Integer, primary_key=True),
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('name', String(length=255, convert_unicode=False,
+                      assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+    )
+
     event = Table(
         'event', meta,
         Column('id', Integer, primary_key=True),
+        Column('stack_id', Integer, ForeignKey("stack.id"), nullable=False),
         Column('created_at', DateTime(timezone=False)),
         Column('updated_at', DateTime(timezone=False)),
         Column('name', String(length=255, convert_unicode=False,
@@ -41,11 +52,12 @@ def upgrade(migrate_engine):
     parsedtemplate = Table(
         'parsed_template', meta,
         Column('id', Integer, primary_key=True),
-        Column('resource_id', Integer()),
+        Column('resource_id', Integer, ForeignKey("resource.id"),\
+              nullable=False),
         Column('template', Text()),
     )
 
-    tables = [rawtemplate, event, resource, parsedtemplate]
+    tables = [rawtemplate, stack, event, resource, parsedtemplate]
     for table in tables:
         try:      
             table.create()
@@ -62,5 +74,5 @@ def downgrade(migrate_engine):
     resource = Table('resource', meta, autoload=True)
     parsedtemplate = Table('parsed_template', meta, autoload=True)
 
-    for table in (rawtemplate, event, resource, parsedtemplate):
+    for table in (rawtemplate, event, stack, parsedtemplate, resource):
         table.drop()

heat/db/sqlalchemy/models.py

@@ -16,16 +16,14 @@ SQLAlchemy models for heat data.
 """
 
 from sqlalchemy import *
-from sqlalchemy.orm import relationship, backref, object_mapper
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.schema import ForeignKeyConstraint
 
-from nova import flags
+from nova import utils
+from heat.db.sqlalchemy.session import get_session
 
-FLAGS = flags.FLAGS
 BASE = declarative_base()
-meta = MetaData()
 
 class HeatBase(object):
     """Base class for Heat Models."""
@@ -85,29 +83,38 @@ class HeatBase(object):
         local.update(joined)
         return local.iteritems()
 
-class RawTemplate(Base, HeatBase):
+class RawTemplate(BASE, HeatBase):
     """Represents an unparsed template which should be in JSON format."""
 
     __tablename__ = 'raw_template'
     id = Column(Integer, primary_key=True)
     template = Text()
 
-class ParsedTemplate(Base, HeatBase):
+class ParsedTemplate(BASE, HeatBase):
     """Represents a parsed template."""
 
     __tablename__ = 'parsed_template'
     id = Column(Integer, primary_key=True) 
     resource_id = Column('resource_id', Integer)
 
-class Event(Base, HeatBase):
+class Stack(BASE, HeatBase):
+    """Represents an generated by the heat engine."""
+
+    __tablename__ = 'stack'
+
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+
+class Event(BASE, HeatBase):
     """Represents an event generated by the heat engine."""
 
     __tablename__ = 'event'
 
     id = Column(Integer, primary_key=True)
+    stack_id = Column(Integer)
     name = Column(String)
  
-class Resource(Base, HeatBase):
+class Resource(BASE, HeatBase):
     """Represents a resource created by the heat engine."""
 
     __tablename__ = 'resource'

heat/db/sqlalchemy/session.py

@@ -0,0 +1,97 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Session Handling for SQLAlchemy backend."""
+
+import sqlalchemy.interfaces
+import sqlalchemy.orm
+from sqlalchemy.exc import DisconnectionError
+
+import nova.exception
+from heat.openstack.common import cfg
+from heat.db import api as db_api
+
+_ENGINE = None
+_MAKER = None
+
+
+def get_session(autocommit=True, expire_on_commit=False):
+    """Return a SQLAlchemy session."""
+    global _ENGINE, _MAKER
+
+    if _MAKER is None or _ENGINE is None:
+        _ENGINE = get_engine()
+        _MAKER = get_maker(_ENGINE, autocommit, expire_on_commit)
+
+    session = _MAKER()
+    session.query = nova.exception.wrap_db_error(session.query)
+    session.flush = nova.exception.wrap_db_error(session.flush)
+    return session
+
+
+class SynchronousSwitchListener(sqlalchemy.interfaces.PoolListener):
+
+    """Switch sqlite connections to non-synchronous mode"""
+
+    def connect(self, dbapi_con, con_record):
+        dbapi_con.execute("PRAGMA synchronous = OFF")
+
+
+class MySQLPingListener(object):
+
+    """
+    Ensures that MySQL connections checked out of the
+    pool are alive.
+
+    Borrowed from:
+    http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f
+    """
+
+    def checkout(self, dbapi_con, con_record, con_proxy):
+        try:
+            dbapi_con.cursor().execute('select 1')
+        except dbapi_con.OperationalError, ex:
+            if ex.args[0] in (2006, 2013, 2014, 2045, 2055):
+                LOG.warn('Got mysql server has gone away: %s', ex)
+                raise DisconnectionError("Database server went away")
+            else:
+                raise
+
+
+def get_engine():
+    """Return a SQLAlchemy engine."""
+    connection_dict = sqlalchemy.engine.url.make_url(_get_sql_connection())
+    engine_args = {
+        "pool_recycle": _get_sql_idle_timeout(),
+        "echo": False,
+        'convert_unicode': True,
+    }
+
+    if 'mysql' in connection_dict.drivername:
+        engine_args['listeners'] = [MySQLPingListener()]
+
+    return sqlalchemy.create_engine(_get_sql_connection(), **engine_args)
+
+
+def get_maker(engine, autocommit=True, expire_on_commit=False):
+    """Return a SQLAlchemy sessionmaker using the given engine."""
+    return sqlalchemy.orm.sessionmaker(bind=engine,
+                                       autocommit=autocommit,
+                                       expire_on_commit=expire_on_commit)
+
+def _get_sql_connection():
+    return db_api.SQL_CONNECTION
+
+def _get_sql_idle_timeout():
+    return db_api.SQL_IDLE_TIMEOUT

heat/engine/api/v1/stacks.py

@@ -49,15 +49,15 @@ class StacksController(object):
         res = {'stacks': [] }
         for s in stack_db:
             mem = {}
-            mem['StackId'] = s
-            mem['StackName'] = s
-            mem['CreationTime'] = 'now'
+            mem['stack_id'] = s
+            mem['stack_name'] = s
+            mem['created_at'] = 'now'
             try:
-                mem['TemplateDescription'] = stack_db[s]['Description']
-                mem['StackStatus'] = stack_db[s]['StackStatus']
+                mem['template_description'] = stack_db[s]['Description']
+                mem['stack_status'] = stack_db[s]['StackStatus']
             except:
-                mem['TemplateDescription'] = 'No description'
-                mem['StackStatus'] = 'unknown'
+                mem['template_description'] = 'No description'
+                mem['stack_status'] = 'unknown'
             res['stacks'].append(mem)
 
         return res
@@ -66,10 +66,10 @@ class StacksController(object):
         res = {'stacks': [] }
         if stack_db.has_key(id):
             mem = {}
-            mem['StackId'] = id
-            mem['StackName'] = id
-            mem['CreationTime'] = 'TODO'
-            mem['LastUpdatedTime'] = 'TODO'
+            mem['stack_id'] = id
+            mem['stack_name'] = id
+            mem['creation_at'] = 'TODO'
+            mem['updated_at'] = 'TODO'
             mem['NotificationARNs'] = 'TODO'
             mem['Outputs'] = [{'Description': 'TODO', 'OutputKey': 'TODO', 'OutputValue': 'TODO' }]
             mem['Parameters'] = stack_db[id]['Parameters']

heat/engine/parser.py

@@ -22,7 +22,7 @@ logger = logging.getLogger('heat.engine.parser')
 
 class Stack:
     def __init__(self, stack_name, template, parms=None):
-
+        self.id = 0
         self.t = template
         if self.t.has_key('Parameters'):
             self.parms = self.t['Parameters']

heat/engine/resources.py

@@ -22,7 +22,7 @@ from novaclient.v1_1 import client
 
 from heat.db import api as db_api
 from heat.common.config import HeatEngineConfigOpts
-
+import pdb
 db_api.configure(HeatEngineConfigOpts())
 
 logger = logging.getLogger('heat.engine.resources')
@@ -82,15 +82,16 @@ class Resource(object):
     def state_set(self, new_state, reason="state changed"):
         if new_state != self.state:
             ev = {}
-            ev['LogicalResourceId'] = self.name
-            ev['PhysicalResourceId'] = self.name
-            ev['StackId'] = self.stack.name
-            ev['StackName'] = self.stack.name
-            ev['ResourceStatus'] = new_state
-            ev['ResourceStatusReason'] = reason
-            ev['ResourceType'] = self.t['Type']
-            ev['ResourceProperties'] = self.t['Properties']
-
+            ev['logical_resource_id'] = self.name
+            ev['physical_resource_id'] = self.name
+            ev['stack_id'] = self.stack.id
+            ev['stack_name'] = self.stack.name
+            ev['resource_status'] = new_state
+            ev['resource_status_reason'] = reason
+            ev['resource_type'] = self.t['Type']
+            ev['resource_properties'] = self.t['Properties']
+            new_stack = db_api.stack_create(None, ev)
+            ev['stack_id'] = new_stack.id
             db_api.event_create(None, ev)
             self.state = new_state
 

tools/README

@@ -1,6 +1,9 @@
 Files in this directory are general developer tools or examples of how
 to do certain activities.
 
+If you're running on F16, make sure you first enable the preview yum repository
+http://fedoraproject.org/wiki/Getting_started_with_OpenStack_on_Fedora_17#Preview_Repository_for_Fedora_16
+
 -----
 Tools
 -----