From 4018da01860671973b41bf4511f96d41f3fc4412 Mon Sep 17 00:00:00 2001 From: Chris Alfonso Date: Thu, 29 Mar 2012 15:14:21 -0400 Subject: [PATCH] Add a heat database to store templates, state, and events Fixes #39 --- MANIFEST.in | 1 + bin/heat-db-setup-fedora | 239 ++++++++++++++++++ etc/heat-engine.conf | 2 + heat.spec | 4 +- heat/db/sqlalchemy/api.py | 35 +++ heat/db/sqlalchemy/manage.py | 5 + heat/db/sqlalchemy/migrate_repo/README | 4 + heat/db/sqlalchemy/migrate_repo/__init__.py | 0 heat/db/sqlalchemy/migrate_repo/manage.py | 5 + heat/db/sqlalchemy/migrate_repo/migrate.cfg | 25 ++ .../migrate_repo/versions/001_norwhal.py | 66 +++++ .../migrate_repo/versions/__init__.py | 0 heat/db/sqlalchemy/models.py | 117 +++++++++ setup.py | 3 +- 14 files changed, 504 insertions(+), 2 deletions(-) create mode 100755 bin/heat-db-setup-fedora create mode 100644 heat/db/sqlalchemy/manage.py create mode 100644 heat/db/sqlalchemy/migrate_repo/README create mode 100644 heat/db/sqlalchemy/migrate_repo/__init__.py create mode 100644 heat/db/sqlalchemy/migrate_repo/manage.py create mode 100644 heat/db/sqlalchemy/migrate_repo/migrate.cfg create mode 100644 heat/db/sqlalchemy/migrate_repo/versions/001_norwhal.py create mode 100644 heat/db/sqlalchemy/migrate_repo/versions/__init__.py create mode 100644 heat/db/sqlalchemy/models.py diff --git a/MANIFEST.in b/MANIFEST.in index a082199a46..ab849f04a6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,6 +7,7 @@ include babel.cfg graft templates include heat/jeos/F16-x86_64-gold-jeos.tdl include heat/jeos/F17-x86_64-gold-jeos.tdl +include heat/db/sqlalchemy/migrate_repo/migrate.cfg graft etc graft docs graft var diff --git a/bin/heat-db-setup-fedora b/bin/heat-db-setup-fedora new file mode 100755 index 0000000000..abbba56a97 --- /dev/null +++ b/bin/heat-db-setup-fedora @@ -0,0 +1,239 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +# +# Print --help output and exit. +# +usage() { + +cat << EOF +Set up a local MySQL database for use with heat. +This script will create a 'heat' database that is accessible +only on localhost by user 'heat' with password 'heat'. + +Usage: heat-db-setup [options] +Options: + --help | -h + Print usage information. + --heatpw | -n + Specify the password for the 'heat' MySQL user that will + use to connect to the 'heat' MySQL database. By default, + the password 'heat' will be used. + --rootpw | -r + Specify the root MySQL password. If the script installs + the MySQL server, it will set the root password to this value + instead of prompting for a password. If the MySQL server is + already installed, this password will be used to connect to the + database instead of having to prompt for it. + --yes | -y + In cases where the script would normally ask for confirmation + before doing something, such as installing mysql-server, + just assume yes. This is useful if you want to run the script + non-interactively. +EOF + + exit 0 +} + +install_mysql_server() { + if [ -z "${ASSUME_YES}" ] ; then + yum install mysql-server + else + yum install -y mysql-server + fi +} + +start_mysql_server() { + systemctl start mysqld.service +} + +MYSQL_HEAT_PW_DEFAULT="heat" +MYSQL_HEAT_PW=${MYSQL_HEAT_PW_DEFAULT} +HEAT_CONFIG="/etc/heat/heat-engine.conf" +ASSUME_YES="" + +while [ $# -gt 0 ] +do + case "$1" in + -h|--help) + usage + ;; + -n|--novapw) + shift + MYSQL_HEAT_PW=${1} + ;; + -r|--rootpw) + shift + MYSQL_ROOT_PW=${1} + ;; + -y|--yes) + ASSUME_YES="yes" + ;; + *) + # ignore + shift + ;; + esac + shift +done + + +# Make sure MySQL is installed. + +NEW_MYSQL_INSTALL=0 +if ! rpm -q mysql-server > /dev/null +then + if [ -z "${ASSUME_YES}" ] ; then + printf "mysql-server is not installed. Would you like to install it now? (y/n): " + read response + case "$response" in + y|Y) + ;; + n|N) + echo "mysql-server must be installed. Please install it before proceeding." + exit 0 + ;; + *) + echo "Invalid response." + exit 1 + esac + fi + + NEW_MYSQL_INSTALL=1 + install_mysql_server +fi + + +# Make sure mysqld is running. + +if ! systemctl status mysqld.service > /dev/null +then + if [ -z "${ASSUME_YES}" ] ; then + printf "mysqld is not running. Would you like to start it now? (y/n): " + read response + case "$response" in + y|Y) + ;; + n|N) + echo "mysqld must be running. Please start it before proceeding." + exit 0 + ;; + *) + echo "Invalid response." + exit 1 + esac + fi + + start_mysql_server + + # If we both installed and started, ensure it starts at boot + [ $NEW_MYSQL_INSTALL -eq 1 ] && chkconfig mysqld on +fi + + +# Get MySQL root access. + +if [ $NEW_MYSQL_INSTALL -eq 1 ] +then + if [ ! "${MYSQL_ROOT_PW+defined}" ] ; then + echo "Since this is a fresh installation of MySQL, please set a password for the 'root' mysql user." + + PW_MATCH=0 + while [ $PW_MATCH -eq 0 ] + do + printf "Enter new password for 'root' mysql user: " + read -s MYSQL_ROOT_PW + echo + printf "Enter new password again: " + read -s PW2 + echo + if [ "${MYSQL_ROOT_PW}" = "${PW2}" ] ; then + PW_MATCH=1 + else + echo "Passwords did not match." + fi + done + fi + + echo "UPDATE mysql.user SET password = password('${MYSQL_ROOT_PW}') WHERE user = 'root'; DELETE FROM mysql.user WHERE user = ''; flush privileges;" | mysql -u root + if ! [ $? -eq 0 ] ; then + echo "Failed to set password for 'root' MySQL user." + exit 1 + fi +elif [ ! "${MYSQL_ROOT_PW+defined}" ] ; then + printf "Please enter the password for the 'root' MySQL user: " + read -s MYSQL_ROOT_PW + echo +fi + + +# Sanity check MySQL credentials. + +MYSQL_ROOT_PW_ARG="" +if [ "${MYSQL_ROOT_PW+defined}" ] +then + MYSQL_ROOT_PW_ARG="--password=${MYSQL_ROOT_PW}" +fi +echo "SELECT 1;" | mysql -u root ${MYSQL_ROOT_PW_ARG} > /dev/null +if ! [ $? -eq 0 ] +then + echo "Failed to connect to the MySQL server. Please check your root user credentials." + exit 1 +fi +echo "Verified connectivity to MySQL." + + +# Now create the db. + +echo "Creating 'heat' database." +cat << EOF | mysql -u root ${MYSQL_ROOT_PW_ARG} +CREATE DATABASE heat; +CREATE USER 'heat'@'localhost' IDENTIFIED BY '${MYSQL_HEAT_PW}'; +CREATE USER 'heat'@'%' IDENTIFIED BY '${MYSQL_HEAT_PW}'; +GRANT ALL ON heat.* TO 'heat'@'localhost'; +GRANT ALL ON heat.* TO 'heat'@'%'; +flush privileges; +EOF + + +# Make sure heat configuration has the right MySQL password. + +if [ "${MYSQL_HEAT_PW}" != "${MYSQL_HEAT_PW_DEFAULT}" ] ; then + echo "Updating 'heat' database password in ${HEAT_CONFIG}" + sed -i -e "s/mysql:\/\/heat:\(.*\)@/mysql:\/\/heat:${MYSQL_HEAT_PW}@/" ${HEAT_CONFIG} +fi + +#create the schema using sqlalchemy-migrate +if test $1 == "rpm"; then + pushd /usr/lib/python2.7/site-packages/heat/db/sqlalchemy +else + pushd /usr/lib/python2.7/site-packages/heat-0.0.1-py2.7.egg/heat/db/sqlalchemy/ +fi + +python migrate_repo/manage.py version_control mysql://heat:heat@localhost/heat migrate_repo +python manage.py upgrade +popd + + +# Do a final sanity check on the database. + +echo "SELECT * FROM migrate_version;" | mysql -u heat --password=${MYSQL_HEAT_PW} heat > /dev/null +if ! [ $? -eq 0 ] +then + echo "Final sanity check failed." + exit 1 +fi + +echo "Complete!" diff --git a/etc/heat-engine.conf b/etc/heat-engine.conf index 12060542a8..c87c4f6a9f 100644 --- a/etc/heat-engine.conf +++ b/etc/heat-engine.conf @@ -23,3 +23,5 @@ use_syslog = False # Facility to use. If unset defaults to LOG_USER. # syslog_log_facility = LOG_LOCAL0 + +sql_connection = mysql://heat:heat@localhost/heat diff --git a/heat.spec b/heat.spec index fc65f60c5a..106484f2da 100644 --- a/heat.spec +++ b/heat.spec @@ -92,6 +92,8 @@ This package contains the OpenStack integration for the Heat project %defattr(-,root,root,-) %{_mandir}/man1/*.gz %{_bindir}/heat +%{_bindir}/heat-db-setup-fedora +%{python_sitelib}/heat/db/* %{python_sitelib}/heat/__init__.* %{python_sitelib}/heat/client.* %{python_sitelib}/heat/cloudformations.* @@ -130,7 +132,7 @@ This package contains the OpenStack integration for the Heat project %{python_sitelib}/heat/engine/client.* %{python_sitelib}/heat/engine/parser.* %{python_sitelib}/heat/engine/resources.* -%{python_sitelib}/heat/engine/simpledb.* +%{python_sitelib}/heat/.* %{python_sitelib}/heat/engine/__init__.* %{python_sitelib}/heat/engine/api/__init__.* %{python_sitelib}/heat/engine/api/v1/__init__.* diff --git a/heat/db/sqlalchemy/api.py b/heat/db/sqlalchemy/api.py index 2bf1ab95d8..3c0ac504af 100644 --- a/heat/db/sqlalchemy/api.py +++ b/heat/db/sqlalchemy/api.py @@ -15,6 +15,41 @@ '''Implementation of SQLAlchemy backend.''' +from nova.db.sqlalchemy.session import get_session +from nova import flags +from nova import utils + +FLAGS = flags.FLAGS + +def model_query(context, *args, **kwargs): + """Query helper that accounts for context's `read_deleted` field. + + :param context: context to query under + :param session: if present, the session to use + :param read_deleted: if present, overrides context's read_deleted field. + :param project_only: if present and context is user-type, then restrict + query to match the context's project_id. + """ + session = kwargs.get('session') or get_session() + read_deleted = kwargs.get('read_deleted') or context.read_deleted + project_only = kwargs.get('project_only') + + query = session.query(*args) + + if read_deleted == 'no': + query = query.filter_by(deleted=False) + elif read_deleted == 'yes': + pass # omit the filter to include deleted and active + elif read_deleted == 'only': + query = query.filter_by(deleted=True) + else: + raise Exception( + _("Unrecognized read_deleted value '%s'") % read_deleted) + + if project_only and is_user_context(context): + query = query.filter_by(project_id=context.project_id) + + return query # a big TODO def raw_template_get(context, template_id): diff --git a/heat/db/sqlalchemy/manage.py b/heat/db/sqlalchemy/manage.py new file mode 100644 index 0000000000..0382d72786 --- /dev/null +++ b/heat/db/sqlalchemy/manage.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python +from migrate.versioning.shell import main + +if __name__ == '__main__': + main(url='mysql://heat:heat@localhost/heat', debug='False', repository='migrate_repo') diff --git a/heat/db/sqlalchemy/migrate_repo/README b/heat/db/sqlalchemy/migrate_repo/README new file mode 100644 index 0000000000..6218f8cac4 --- /dev/null +++ b/heat/db/sqlalchemy/migrate_repo/README @@ -0,0 +1,4 @@ +This is a database migration repository. + +More information at +http://code.google.com/p/sqlalchemy-migrate/ diff --git a/heat/db/sqlalchemy/migrate_repo/__init__.py b/heat/db/sqlalchemy/migrate_repo/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/heat/db/sqlalchemy/migrate_repo/manage.py b/heat/db/sqlalchemy/migrate_repo/manage.py new file mode 100644 index 0000000000..39fa3892e5 --- /dev/null +++ b/heat/db/sqlalchemy/migrate_repo/manage.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python +from migrate.versioning.shell import main + +if __name__ == '__main__': + main(debug='False') diff --git a/heat/db/sqlalchemy/migrate_repo/migrate.cfg b/heat/db/sqlalchemy/migrate_repo/migrate.cfg new file mode 100644 index 0000000000..134fc065d0 --- /dev/null +++ b/heat/db/sqlalchemy/migrate_repo/migrate.cfg @@ -0,0 +1,25 @@ +[db_settings] +# Used to identify which repository this database is versioned under. +# You can use the name of your project. +repository_id=heat + +# The name of the database table used to track the schema version. +# This name shouldn't already be used by your project. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. +version_table=migrate_version + +# When committing a change script, Migrate will attempt to generate the +# sql for all supported databases; normally, if one of them fails - probably +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually +# be using to ensure your updates to that database work properly. +# This must be a list; example: ['postgres','sqlite'] +required_dbs=[] + +# When creating new change scripts, Migrate will stamp the new script with +# a version number. By default this is latest_version + 1. You can set this +# to 'true' to tell Migrate to use the UTC timestamp instead. +use_timestamp_numbering=False diff --git a/heat/db/sqlalchemy/migrate_repo/versions/001_norwhal.py b/heat/db/sqlalchemy/migrate_repo/versions/001_norwhal.py new file mode 100644 index 0000000000..852da9fdad --- /dev/null +++ b/heat/db/sqlalchemy/migrate_repo/versions/001_norwhal.py @@ -0,0 +1,66 @@ +from sqlalchemy import * +from migrate import * + +def upgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + rawtemplate = Table( + 'raw_template', meta, + Column('id', Integer, primary_key=True), + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('template', Text()), + ) + + event = Table( + 'event', meta, + Column('id', Integer, primary_key=True), + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('name', String(length=255, convert_unicode=False, + assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + ) + + resource = Table( + 'resource', meta, + Column('id', Integer, primary_key=True), + Column('instance_id', String(length=255, convert_unicode=False, + assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('state', Integer()), + Column('state_description', String(length=255, convert_unicode=False, + assert_unicode=None, + unicode_error=None, + _warn_on_bytestring=False)), + ) + + parsedtemplate = Table( + 'parsed_template', meta, + Column('id', Integer, primary_key=True), + Column('resource_id', Integer()), + Column('template', Text()), + ) + + tables = [rawtemplate, event, resource, parsedtemplate] + for table in tables: + try: + table.create() + except Exception: + meta.drop_all(tables=tables) + raise + +def downgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + rawtemplate = Table('raw_template', meta, autoload=True) + event = Table('event', meta, autoload=True) + resource = Table('resource', meta, autoload=True) + parsedtemplate = Table('parsed_template', meta, autoload=True) + + for table in (rawtemplate, event, resource, parsedtemplate): + table.drop() diff --git a/heat/db/sqlalchemy/migrate_repo/versions/__init__.py b/heat/db/sqlalchemy/migrate_repo/versions/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/heat/db/sqlalchemy/models.py b/heat/db/sqlalchemy/models.py new file mode 100644 index 0000000000..886306bdb3 --- /dev/null +++ b/heat/db/sqlalchemy/models.py @@ -0,0 +1,117 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +SQLAlchemy models for heat data. +""" + +from sqlalchemy import * +from sqlalchemy.orm import relationship, backref, object_mapper +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.schema import ForeignKeyConstraint + +from nova import flags + +FLAGS = flags.FLAGS +BASE = declarative_base() +meta = MetaData() + +class HeatBase(object): + """Base class for Heat Models.""" + __table_args__ = {'mysql_engine': 'InnoDB'} + __table_initialized__ = False + created_at = Column(DateTime, default=utils.utcnow) + updated_at = Column(DateTime, onupdate=utils.utcnow) + + def save(self, session=None): + """Save this object.""" + if not session: + session = get_session() + session.add(self) + try: + session.flush() + except IntegrityError, e: + if str(e).endswith('is not unique'): + raise exception.Duplicate(str(e)) + else: + raise + + def delete(self, session=None): + """Delete this object.""" + self.deleted = True + self.deleted_at = utils.utcnow() + self.save(session=session) + + def __setitem__(self, key, value): + setattr(self, key, value) + + def __getitem__(self, key): + return getattr(self, key) + + def get(self, key, default=None): + return getattr(self, key, default) + + def __iter__(self): + self._i = iter(object_mapper(self).columns) + return self + + def next(self): + n = self._i.next().name + return n, getattr(self, n) + + def update(self, values): + """Make the model object behave like a dict""" + for k, v in values.iteritems(): + setattr(self, k, v) + + def iteritems(self): + """Make the model object behave like a dict. + + Includes attributes from joins.""" + local = dict(self) + joined = dict([(k, v) for k, v in self.__dict__.iteritems() + if not k[0] == '_']) + local.update(joined) + return local.iteritems() + +class RawTemplate(Base, HeatBase): + """Represents an unparsed template which should be in JSON format.""" + + __tablename__ = 'raw_template' + id = Column(Integer, primary_key=True) + template = Text() + +class ParsedTemplate(Base, HeatBase): + """Represents a parsed template.""" + + __tablename__ = 'parsed_template' + id = Column(Integer, primary_key=True) + resource_id = Column('resource_id', Integer) + +class Event(Base, HeatBase): + """Represents an event generated by the heat engine.""" + + __tablename__ = 'event' + + id = Column(Integer, primary_key=True) + name = Column(String) + +class Resource(Base, HeatBase): + """Represents a resource created by the heat engine.""" + + __tablename__ = 'resource' + + id = Column(Integer, primary_key=True) + state = Column(String) + state_description = Column('state_description', String) diff --git a/setup.py b/setup.py index 52d95847f4..09809bbc4a 100755 --- a/setup.py +++ b/setup.py @@ -89,7 +89,8 @@ setup( ], scripts=['bin/heat', 'bin/heat-api', - 'bin/heat-engine'], + 'bin/heat-engine', + 'bin/heat-db-setup-fedora'], data_files=[('/etc/heat', ['etc/heat-api.conf', 'etc/heat-api-paste.ini', 'etc/heat-engine.conf',