Add a heat database to store templates, state, and events

Fixes #39

MANIFEST.in

@@ -7,6 +7,7 @@ include babel.cfg
 graft templates
 include heat/jeos/F16-x86_64-gold-jeos.tdl
 include heat/jeos/F17-x86_64-gold-jeos.tdl
+include heat/db/sqlalchemy/migrate_repo/migrate.cfg
 graft etc
 graft docs
 graft var

bin/heat-db-setup-fedora

@@ -0,0 +1,239 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Print --help output and exit.
+#
+usage() {
+
+cat << EOF
+Set up a local MySQL database for use with heat.
+This script will create a 'heat' database that is accessible
+only on localhost by user 'heat' with password 'heat'.
+
+Usage: heat-db-setup [options]
+Options:
+	--help        | -h
+		Print usage information.
+	--heatpw <pw> | -n <pw>
+		Specify the password for the 'heat' MySQL user that  will
+		use to connect to the 'heat' MySQL database.  By default,
+		the password 'heat' will be used.
+	--rootpw <pw> | -r <pw>
+		Specify the root MySQL password.  If the script installs
+		the MySQL server, it will set the root password to this value
+		instead of prompting for a password.  If the MySQL server is
+		already installed, this password will be used to connect to the
+		database instead of having to prompt for it.
+	--yes         | -y
+		In cases where the script would normally ask for confirmation
+		before doing something, such as installing mysql-server,
+		just assume yes.  This is useful if you want to run the script
+		non-interactively.
+EOF
+
+	exit 0
+}
+
+install_mysql_server() {
+	if [ -z "${ASSUME_YES}" ] ; then
+		yum install mysql-server
+	else
+		yum install -y mysql-server
+	fi
+}
+
+start_mysql_server() {
+	systemctl start mysqld.service
+}
+
+MYSQL_HEAT_PW_DEFAULT="heat"
+MYSQL_HEAT_PW=${MYSQL_HEAT_PW_DEFAULT}
+HEAT_CONFIG="/etc/heat/heat-engine.conf"
+ASSUME_YES=""
+
+while [ $# -gt 0 ]
+do
+	case "$1" in
+		-h|--help)
+			usage
+			;;
+		-n|--novapw)
+			shift
+			MYSQL_HEAT_PW=${1}
+			;;
+		-r|--rootpw)
+			shift
+			MYSQL_ROOT_PW=${1}
+			;;
+		-y|--yes)
+			ASSUME_YES="yes"
+			;;
+		*)
+			# ignore
+			shift
+			;;
+	esac
+	shift
+done
+
+
+# Make sure MySQL is installed.
+
+NEW_MYSQL_INSTALL=0
+if ! rpm -q mysql-server > /dev/null
+then
+	if [ -z "${ASSUME_YES}" ] ; then
+		printf "mysql-server is not installed.  Would you like to install it now? (y/n): "
+		read response
+		case "$response" in
+			y|Y)
+				;;
+			n|N)
+				echo "mysql-server must be installed.  Please install it before proceeding."
+				exit 0
+				;;
+			*)
+				echo "Invalid response."
+				exit 1
+		esac
+	fi
+
+	NEW_MYSQL_INSTALL=1
+	install_mysql_server
+fi
+
+
+# Make sure mysqld is running.
+
+if ! systemctl status mysqld.service > /dev/null
+then
+	if [ -z "${ASSUME_YES}" ] ; then
+		printf "mysqld is not running.  Would you like to start it now? (y/n): "
+		read response
+		case "$response" in
+			y|Y)
+				;;
+			n|N)
+				echo "mysqld must be running.  Please start it before proceeding."
+				exit 0
+				;;
+			*)
+				echo "Invalid response."
+				exit 1
+		esac
+	fi
+
+	start_mysql_server
+
+	# If we both installed and started, ensure it starts at boot
+	[ $NEW_MYSQL_INSTALL -eq 1 ] && chkconfig mysqld on
+fi
+
+
+# Get MySQL root access.
+
+if [ $NEW_MYSQL_INSTALL -eq 1 ]
+then
+	if [ ! "${MYSQL_ROOT_PW+defined}" ] ; then
+		echo "Since this is a fresh installation of MySQL, please set a password for the 'root' mysql user."
+
+		PW_MATCH=0
+		while [ $PW_MATCH -eq 0 ]
+		do
+			printf "Enter new password for 'root' mysql user: "
+			read -s MYSQL_ROOT_PW
+			echo
+			printf "Enter new password again: "
+			read -s PW2
+			echo
+			if [ "${MYSQL_ROOT_PW}" = "${PW2}" ] ; then
+				PW_MATCH=1
+			else
+				echo "Passwords did not match."
+			fi
+		done
+	fi
+
+	echo "UPDATE mysql.user SET password = password('${MYSQL_ROOT_PW}') WHERE user = 'root'; DELETE FROM mysql.user WHERE user = ''; flush privileges;" | mysql -u root
+	if ! [ $? -eq 0 ] ; then
+		echo "Failed to set password for 'root' MySQL user."
+		exit 1
+	fi
+elif [ ! "${MYSQL_ROOT_PW+defined}" ] ; then
+	printf "Please enter the password for the 'root' MySQL user: "
+	read -s MYSQL_ROOT_PW
+	echo
+fi
+
+
+# Sanity check MySQL credentials.
+
+MYSQL_ROOT_PW_ARG=""
+if [ "${MYSQL_ROOT_PW+defined}" ]
+then
+	MYSQL_ROOT_PW_ARG="--password=${MYSQL_ROOT_PW}"
+fi
+echo "SELECT 1;" | mysql -u root ${MYSQL_ROOT_PW_ARG} > /dev/null
+if ! [ $? -eq 0 ]
+then
+	echo "Failed to connect to the MySQL server.  Please check your root user credentials."
+	exit 1
+fi
+echo "Verified connectivity to MySQL."
+
+
+# Now create the db.
+
+echo "Creating 'heat' database."
+cat << EOF | mysql -u root ${MYSQL_ROOT_PW_ARG}
+CREATE DATABASE heat;
+CREATE USER 'heat'@'localhost' IDENTIFIED BY '${MYSQL_HEAT_PW}';
+CREATE USER 'heat'@'%' IDENTIFIED BY '${MYSQL_HEAT_PW}';
+GRANT ALL ON heat.* TO 'heat'@'localhost';
+GRANT ALL ON heat.* TO 'heat'@'%';
+flush privileges;
+EOF
+
+
+# Make sure heat configuration has the right MySQL password.
+
+if [ "${MYSQL_HEAT_PW}" != "${MYSQL_HEAT_PW_DEFAULT}" ] ; then
+	echo "Updating 'heat' database password in ${HEAT_CONFIG}"
+	sed -i -e "s/mysql:\/\/heat:\(.*\)@/mysql:\/\/heat:${MYSQL_HEAT_PW}@/" ${HEAT_CONFIG}
+fi
+
+#create the schema using sqlalchemy-migrate
+if test $1 == "rpm"; then
+  pushd /usr/lib/python2.7/site-packages/heat/db/sqlalchemy
+else
+  pushd /usr/lib/python2.7/site-packages/heat-0.0.1-py2.7.egg/heat/db/sqlalchemy/
+fi
+
+python migrate_repo/manage.py version_control mysql://heat:heat@localhost/heat migrate_repo
+python manage.py upgrade
+popd
+
+
+# Do a final sanity check on the database.
+
+echo "SELECT * FROM migrate_version;" | mysql -u heat --password=${MYSQL_HEAT_PW} heat > /dev/null
+if ! [ $? -eq 0 ]
+then
+	echo "Final sanity check failed."
+	exit 1
+fi
+
+echo "Complete!"

etc/heat-engine.conf

@@ -23,3 +23,5 @@ use_syslog = False
 
 # Facility to use. If unset defaults to LOG_USER.
 # syslog_log_facility = LOG_LOCAL0
+
+sql_connection = mysql://heat:heat@localhost/heat

heat.spec

@@ -92,6 +92,8 @@ This package contains the OpenStack integration for the Heat project
 %defattr(-,root,root,-)
 %{_mandir}/man1/*.gz
 %{_bindir}/heat
+%{_bindir}/heat-db-setup-fedora
+%{python_sitelib}/heat/db/*
 %{python_sitelib}/heat/__init__.*
 %{python_sitelib}/heat/client.*
 %{python_sitelib}/heat/cloudformations.*
@@ -130,7 +132,7 @@ This package contains the OpenStack integration for the Heat project
 %{python_sitelib}/heat/engine/client.*
 %{python_sitelib}/heat/engine/parser.*
 %{python_sitelib}/heat/engine/resources.*
-%{python_sitelib}/heat/engine/simpledb.*
+%{python_sitelib}/heat/.*
 %{python_sitelib}/heat/engine/__init__.*
 %{python_sitelib}/heat/engine/api/__init__.*
 %{python_sitelib}/heat/engine/api/v1/__init__.*

heat/db/sqlalchemy/api.py

@@ -15,6 +15,41 @@
 
 '''Implementation of SQLAlchemy backend.'''
 
+from nova.db.sqlalchemy.session import get_session
+from nova import flags
+from nova import utils
+
+FLAGS = flags.FLAGS
+
+def model_query(context, *args, **kwargs):
+    """Query helper that accounts for context's `read_deleted` field.
+
+    :param context: context to query under
+    :param session: if present, the session to use
+    :param read_deleted: if present, overrides context's read_deleted field.
+    :param project_only: if present and context is user-type, then restrict
+            query to match the context's project_id.
+    """
+    session = kwargs.get('session') or get_session()
+    read_deleted = kwargs.get('read_deleted') or context.read_deleted
+    project_only = kwargs.get('project_only')
+
+    query = session.query(*args)
+
+    if read_deleted == 'no':
+        query = query.filter_by(deleted=False)
+    elif read_deleted == 'yes':
+        pass  # omit the filter to include deleted and active
+    elif read_deleted == 'only':
+        query = query.filter_by(deleted=True)
+    else:
+        raise Exception(
+                _("Unrecognized read_deleted value '%s'") % read_deleted)
+
+    if project_only and is_user_context(context):
+        query = query.filter_by(project_id=context.project_id)
+
+    return query
 
 # a big TODO
 def raw_template_get(context, template_id):

heat/db/sqlalchemy/manage.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+from migrate.versioning.shell import main
+
+if __name__ == '__main__':
+    main(url='mysql://heat:heat@localhost/heat', debug='False', repository='migrate_repo')

heat/db/sqlalchemy/migrate_repo/README

@@ -0,0 +1,4 @@
+This is a database migration repository.
+
+More information at
+http://code.google.com/p/sqlalchemy-migrate/

heat/db/sqlalchemy/migrate_repo/manage.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+from migrate.versioning.shell import main
+
+if __name__ == '__main__':
+    main(debug='False')

heat/db/sqlalchemy/migrate_repo/migrate.cfg

@@ -0,0 +1,25 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=heat
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to 
+# change the table name in each database too. 
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the 
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the 
+# commit continues, perhaps ending successfully. 
+# Databases in this list MUST compile successfully during a commit, or the 
+# entire commit will fail. List the databases your application will actually 
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]
+
+# When creating new change scripts, Migrate will stamp the new script with
+# a version number. By default this is latest_version + 1. You can set this
+# to 'true' to tell Migrate to use the UTC timestamp instead.
+use_timestamp_numbering=False

heat/db/sqlalchemy/migrate_repo/versions/001_norwhal.py

@@ -0,0 +1,66 @@
+from sqlalchemy import *
+from migrate import *
+
+def upgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+    rawtemplate = Table(
+        'raw_template', meta,
+        Column('id', Integer, primary_key=True),
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('template', Text()),
+    )
+
+    event = Table(
+        'event', meta,
+        Column('id', Integer, primary_key=True),
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('name', String(length=255, convert_unicode=False,
+                      assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+    )
+
+    resource = Table(
+        'resource', meta,
+        Column('id', Integer, primary_key=True),
+        Column('instance_id', String(length=255, convert_unicode=False,
+              assert_unicode=None,
+              unicode_error=None, _warn_on_bytestring=False)),
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('state', Integer()),
+        Column('state_description', String(length=255, convert_unicode=False,
+                                           assert_unicode=None,
+                                           unicode_error=None, 
+                                           _warn_on_bytestring=False)),
+    )
+
+    parsedtemplate = Table(
+        'parsed_template', meta,
+        Column('id', Integer, primary_key=True),
+        Column('resource_id', Integer()),
+        Column('template', Text()),
+    )
+
+    tables = [rawtemplate, event, resource, parsedtemplate]
+    for table in tables:
+        try:      
+            table.create()
+        except Exception:
+            meta.drop_all(tables=tables)
+            raise
+
+def downgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+    rawtemplate = Table('raw_template', meta, autoload=True)
+    event = Table('event', meta, autoload=True)
+    resource = Table('resource', meta, autoload=True)
+    parsedtemplate = Table('parsed_template', meta, autoload=True)
+
+    for table in (rawtemplate, event, resource, parsedtemplate):
+        table.drop()

heat/db/sqlalchemy/models.py

@@ -0,0 +1,117 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+SQLAlchemy models for heat data.
+"""
+
+from sqlalchemy import *
+from sqlalchemy.orm import relationship, backref, object_mapper
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.schema import ForeignKeyConstraint
+
+from nova import flags
+
+FLAGS = flags.FLAGS
+BASE = declarative_base()
+meta = MetaData()
+
+class HeatBase(object):
+    """Base class for Heat Models."""
+    __table_args__ = {'mysql_engine': 'InnoDB'}
+    __table_initialized__ = False
+    created_at = Column(DateTime, default=utils.utcnow)
+    updated_at = Column(DateTime, onupdate=utils.utcnow)
+
+    def save(self, session=None):
+        """Save this object."""
+        if not session:
+            session = get_session()
+        session.add(self)
+        try:
+            session.flush()
+        except IntegrityError, e:
+            if str(e).endswith('is not unique'):
+                raise exception.Duplicate(str(e))
+            else:
+                raise
+
+    def delete(self, session=None):
+        """Delete this object."""
+        self.deleted = True
+        self.deleted_at = utils.utcnow()
+        self.save(session=session)
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def get(self, key, default=None):
+        return getattr(self, key, default)
+
+    def __iter__(self):
+        self._i = iter(object_mapper(self).columns)
+        return self
+
+    def next(self):
+        n = self._i.next().name
+        return n, getattr(self, n)
+
+    def update(self, values):
+        """Make the model object behave like a dict"""
+        for k, v in values.iteritems():
+            setattr(self, k, v)
+
+    def iteritems(self):
+        """Make the model object behave like a dict.
+
+        Includes attributes from joins."""
+        local = dict(self)
+        joined = dict([(k, v) for k, v in self.__dict__.iteritems()
+                      if not k[0] == '_'])
+        local.update(joined)
+        return local.iteritems()
+
+class RawTemplate(Base, HeatBase):
+    """Represents an unparsed template which should be in JSON format."""
+
+    __tablename__ = 'raw_template'
+    id = Column(Integer, primary_key=True)
+    template = Text()
+
+class ParsedTemplate(Base, HeatBase):
+    """Represents a parsed template."""
+
+    __tablename__ = 'parsed_template'
+    id = Column(Integer, primary_key=True) 
+    resource_id = Column('resource_id', Integer)
+
+class Event(Base, HeatBase):
+    """Represents an event generated by the heat engine."""
+
+    __tablename__ = 'event'
+
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+ 
+class Resource(Base, HeatBase):
+    """Represents a resource created by the heat engine."""
+
+    __tablename__ = 'resource'
+
+    id = Column(Integer, primary_key=True)
+    state = Column(String)
+    state_description = Column('state_description', String)

setup.py

@@ -89,7 +89,8 @@ setup(
     ],
     scripts=['bin/heat',
              'bin/heat-api',
-             'bin/heat-engine'],
+             'bin/heat-engine',
+             'bin/heat-db-setup-fedora'],
     data_files=[('/etc/heat', ['etc/heat-api.conf',
                                'etc/heat-api-paste.ini',
                                'etc/heat-engine.conf',