From 5f1fbbee3c46177e43acafe107d5b9aa712296c4 Mon Sep 17 00:00:00 2001 From: Bryan Strassner Date: Thu, 5 Apr 2018 10:44:41 -0500 Subject: [PATCH] [396582] Add alembic support to Deckhand Updates Deckhand to use alembic to manage database upgrades. Moves from creating tables at startup of Deckhand to the db-sync job. Change-Id: I6f4cb237fadc46fbee81d1c33096f48a720f589f --- alembic.ini | 74 +++++++ alembic/README | 1 + alembic/env.py | 95 ++++++++ alembic/script.py.mako | 24 ++ .../918bbfd28185_initial_deckhand_base.py | 208 ++++++++++++++++++ charts/deckhand/templates/bin/_db-sync.sh.tpl | 19 +- charts/deckhand/templates/job-db-sync.yaml | 4 +- charts/deckhand/values.yaml | 2 +- deckhand/db/sqlalchemy/api.py | 4 +- deckhand/db/sqlalchemy/models.py | 19 +- deckhand/tests/unit/base.py | 2 +- docs/source/getting-started.rst | 63 ++++++ docs/source/glossary.rst | 28 +++ entrypoint.sh | 35 +-- images/deckhand/Dockerfile | 2 + requirements.txt | 1 + tools/functional-tests.sh | 7 + tox.ini | 2 +- 18 files changed, 566 insertions(+), 24 deletions(-) create mode 100644 alembic.ini create mode 100644 alembic/README create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/918bbfd28185_initial_deckhand_base.py diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 00000000..9a3b6ad3 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,74 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# timezone to use when rendering the date +# within the migration file as well as the filename. +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat alembic/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = set in alembic/env.py + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = console +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = console +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = INFO +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/README b/alembic/README new file mode 100644 index 00000000..98e4f9c4 --- /dev/null +++ b/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 00000000..9cb256e1 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,95 @@ +# Generated file from Alembic, modified portions copyright follows: +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import with_statement +import logging +from logging.config import fileConfig +import os + +from alembic import context +from oslo_config import cfg +from sqlalchemy import engine_from_config, pool + +from deckhand.db.sqlalchemy import api as db_api +from deckhand.db.sqlalchemy import models + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + + +# Portions modified for Deckhand Specifics: +# Set up and retrieve the config file for Deckhand. Sets up the oslo_config +logger = logging.getLogger('alembic.env') +CONF = cfg.CONF +dirname = os.environ.get('DECKHAND_CONFIG_DIR', '/etc/deckhand').strip() +config_files = [os.path.join(dirname, 'deckhand.conf')] +CONF([], project='deckhand', default_config_files=config_files) +logger.info("Database Connection: %s", CONF.database.connection) +config.set_main_option('sqlalchemy.url', CONF.database.connection) +models.register_models(db_api.get_engine(), + CONF.database.connection) +target_metadata = models.BASE.metadata +# End Deckhand Specifics + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 00000000..2c015630 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/918bbfd28185_initial_deckhand_base.py b/alembic/versions/918bbfd28185_initial_deckhand_base.py new file mode 100644 index 00000000..42bc0351 --- /dev/null +++ b/alembic/versions/918bbfd28185_initial_deckhand_base.py @@ -0,0 +1,208 @@ +"""initial deckhand base + +Revision ID: 918bbfd28185 +Revises: +Create Date: 2018-04-04 17:19:24.222703 + +""" +import logging + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql +from sqlalchemy.sql import text + +# revision identifiers, used by Alembic. +revision = '918bbfd28185' +down_revision = None +branch_labels = None +depends_on = None + +LOG = logging.getLogger('alembic.runtime.migration') + +tables_select = text(""" +select table_name from information_schema.tables where table_schema = 'public' + and table_name in ('buckets', 'revisions','documents', 'revision_tags', + 'validations') +""") + +check_documents_columns = text(""" +select column_name from information_schema.columns + where table_name = 'documents' and column_name in ('_metadata', 'layer') +""") + +get_constraints = text(""" +select conname from pg_constraint +""") + +convert_layer = text(""" +update documents d1 set layer = ( + select meta->'layeringDefinition'->>'layer' from documents d2 + where d2.id = d1.id) +""") + + +def upgrade(): + + # Need to check if the tables exist first. + # If they do, then we can't create them, and rather need to: + # check if documents has _metadata or meta column + # rename to meta if it does. + # check if documents.layer exists + # if not, add it and populate it from + # metadata.layeringDefinition.layer in the associated document + # If the tables don't exist it is a new environment; create tables. + # + # Note that this is not fool-proof, if we have environments that are + # not in a state accounted for in this migration, the migration will fail + # This is easist and best if this first migration is starting from an + # empty database. + # + # IMPORTANT Note: + # It is irregular for migrations to conditionally apply changes. + # Migraitons are generally straightforward applicaiton of changes -- e.g. + # crate tables, drop columns, etc... + # Do not model future migrations after this migration, which is specially + # crafted to coerce non-Alembic manageed databases into an Alembic-managed + # form. + + conn = op.get_bind() + LOG.info("Finding tables with query: %s", tables_select) + rs = conn.execute(tables_select) + existing_tables = [row[0] for row in rs] + LOG.info("Existing tables: %s", str(existing_tables)) + + if 'buckets' not in existing_tables: + LOG.info("'buckets' not present. Creating table") + op.create_table('buckets', + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.Column('deleted', sa.Boolean(), nullable=False), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=36), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name'), + mysql_charset='utf8', + mysql_engine='Postgre' + ) + + if 'revisions' not in existing_tables: + LOG.info("'revisions' not present. Creating table") + op.create_table('revisions', + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.Column('deleted', sa.Boolean(), nullable=False), + sa.Column('id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id'), + mysql_charset='utf8', + mysql_engine='Postgre' + ) + + if 'documents' not in existing_tables: + LOG.info("'documents' not present. Creating table") + op.create_table('documents', + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.Column('deleted', sa.Boolean(), nullable=False), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=64), nullable=False), + sa.Column('schema', sa.String(length=64), nullable=False), + sa.Column('layer', sa.String(length=64), nullable=True), + sa.Column('meta', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('data', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('data_hash', sa.String(), nullable=False), + sa.Column('metadata_hash', sa.String(), nullable=False), + sa.Column('bucket_id', sa.Integer(), nullable=False), + sa.Column('revision_id', sa.Integer(), nullable=False), + sa.Column('orig_revision_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['bucket_id'], ['buckets.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['orig_revision_id'], ['revisions.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['revision_id'], ['revisions.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('schema', 'layer', 'name', 'revision_id', name='duplicate_document_constraint') + ) + else: + # documents has undergone some changes that need to be accounted for + # in this migration to ensure a common base. + LOG.info("Finding columns in 'documents' table with query: %s", + check_documents_columns) + rs = conn.execute(check_documents_columns) + columns = [row[0] for row in rs] + LOG.info("Columns are: %s", str(columns)) + + if '_metadata' in columns: + LOG.info("Found '_metadata' column; will rename to 'meta'") + op.alter_column('documents', '_metadata', nullable=False, + new_column_name='meta') + LOG.info("'_metadata' renamed to 'meta'") + if 'layer' not in columns: + LOG.info("'layer' column is not present. Adding column and" + " extracting data from meta column") + + # remove the constraint that is being modified + rs = conn.execute(get_constraints) + constraints = [row[0] for row in rs] + + if 'duplicate_document_constraint' in constraints: + op.drop_constraint('duplicate_document_constraint', + 'documents') + + # add the layer column to documents + op.add_column('documents', + sa.Column('layer', sa.String(length=64), nullable=True) + ) + + # convert the data from meta to here. + conn.execute(convert_layer) + + # add the constraint back in with the wole set of columns + op.create_unique_constraint('duplicate_document_constraint', + 'documents', ['schema', 'layer', 'name', 'revision_id'] + ) + LOG.info("'layer' column added and initialized") + + if 'revision_tags' not in existing_tables: + LOG.info("'revision_tags' not present. Creating table") + op.create_table('revision_tags', + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.Column('deleted', sa.Boolean(), nullable=False), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('tag', sa.String(length=64), nullable=False), + sa.Column('data', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('revision_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['revision_id'], ['revisions.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + mysql_charset='utf8', + mysql_engine='Postgre' + ) + + if 'validations' not in existing_tables: + LOG.info("'validations' not present. Creating table") + op.create_table('validations', + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.Column('deleted', sa.Boolean(), nullable=False), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=64), nullable=False), + sa.Column('status', sa.String(length=8), nullable=False), + sa.Column('validator', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('errors', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('revision_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['revision_id'], ['revisions.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + mysql_charset='utf8', + mysql_engine='Postgre' + ) + +def downgrade(): + op.drop_table('validations') + op.drop_table('revision_tags') + op.drop_table('documents') + op.drop_table('revisions') + op.drop_table('buckets') diff --git a/charts/deckhand/templates/bin/_db-sync.sh.tpl b/charts/deckhand/templates/bin/_db-sync.sh.tpl index 8d4d26ea..ea6f169d 100644 --- a/charts/deckhand/templates/bin/_db-sync.sh.tpl +++ b/charts/deckhand/templates/bin/_db-sync.sh.tpl @@ -1,6 +1,21 @@ #!/bin/bash -# Pending inputs on what need to be done for db-sync + +{{/* +Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} set -ex -export HOME=/tmp +alembic upgrade head diff --git a/charts/deckhand/templates/job-db-sync.yaml b/charts/deckhand/templates/job-db-sync.yaml index 300506e4..5098ec20 100644 --- a/charts/deckhand/templates/job-db-sync.yaml +++ b/charts/deckhand/templates/job-db-sync.yaml @@ -40,8 +40,8 @@ spec: {{ tuple $envAll $dependencies $mounts_deckhand_db_sync_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - name: deckhand-db-sync - image: {{ .Values.images.tags.db_sync }} - imagePullPolicy: {{ .Values.images.pull_policy }} + image: {{ .Values.images.tags.db_sync | quote }} + imagePullPolicy: {{ .Values.images.pull_policy | quote }} {{ tuple $envAll $envAll.Values.pod.resources.jobs.db_sync | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} env: - name: DECKHAND_DB_URL diff --git a/charts/deckhand/values.yaml b/charts/deckhand/values.yaml index 8c6bc828..66f6a58f 100644 --- a/charts/deckhand/values.yaml +++ b/charts/deckhand/values.yaml @@ -23,7 +23,7 @@ images: deckhand: quay.io/attcomdev/deckhand:latest dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.0 db_init: docker.io/postgres:9.5 - db_sync: docker.io/postgres:9.5 + db_sync: quay.io/attcomdev/deckhand:latest ks_endpoints: docker.io/openstackhelm/heat:newton ks_service: docker.io/openstackhelm/heat:newton ks_user: docker.io/openstackhelm/heat:newton diff --git a/deckhand/db/sqlalchemy/api.py b/deckhand/db/sqlalchemy/api.py index f3a34ac8..f3b0783b 100644 --- a/deckhand/db/sqlalchemy/api.py +++ b/deckhand/db/sqlalchemy/api.py @@ -68,8 +68,10 @@ def drop_db(): models.unregister_models(get_engine()) -def setup_db(connection_string): +def setup_db(connection_string, create_tables=False): models.register_models(get_engine(), connection_string) + if create_tables: + models.create_tables(get_engine()) def raw_query(query, **kwargs): diff --git a/deckhand/db/sqlalchemy/models.py b/deckhand/db/sqlalchemy/models.py index 100e5709..5abd5c6b 100644 --- a/deckhand/db/sqlalchemy/models.py +++ b/deckhand/db/sqlalchemy/models.py @@ -211,15 +211,30 @@ def __build_tables(blob_type_obj, blob_type_list): def register_models(engine, connection_string): - global BASE + """Register the sqlalchemy tables itno the BASE.metadata + Sets up the database model objects. Does not create the tables in + the associated configured database. (see create_tables) + """ blob_types = ((JSONB, JSONB) if 'postgresql' in connection_string else (PickleType, oslo_types.JsonEncodedList())) - LOG.debug('Instantiating DB tables using %s, %s as the column type ' + LOG.debug('Initializing DB tables using %s, %s as the column type ' 'for dictionaries, lists.', *blob_types) __build_tables(*blob_types) + + +def create_tables(engine): + """Creates database tables for all models with the given engine. + + This will be done only by tests that do not have their tables + set up by Alembic running during the associated helm chart db_sync job. + """ + global BASE + + LOG.debug('Creating DB tables') + BASE.metadata.create_all(engine) diff --git a/deckhand/tests/unit/base.py b/deckhand/tests/unit/base.py index dc999e5a..980aeaa3 100644 --- a/deckhand/tests/unit/base.py +++ b/deckhand/tests/unit/base.py @@ -112,5 +112,5 @@ class DeckhandWithDBTestCase(DeckhandTestCase): self.override_config( 'connection', os.environ.get('PIFPAF_URL', 'sqlite://'), group='database') - db_api.setup_db(CONF.database.connection) + db_api.setup_db(CONF.database.connection, create_tables=True) self.addCleanup(db_api.drop_db) diff --git a/docs/source/getting-started.rst b/docs/source/getting-started.rst index bd8ee81b..c74a63f9 100644 --- a/docs/source/getting-started.rst +++ b/docs/source/getting-started.rst @@ -96,6 +96,14 @@ Substitute the connection information (which can be retrieved by running # (string value) connection = postgresql://localhost/postgres?host=/tmp/tmpsg6tn3l9&port=9824 +Run an update to the Database to bring it to the current code level:: + + $ [sudo] docker run --rm \ + --net=host \ + -v $CONF_DIR:/etc/deckhand \ + quay.io/attcomdev/deckhand:latest \ + alembic upgrade head + Finally, run Deckhand via Docker:: $ [sudo] docker run --rm \ @@ -211,6 +219,61 @@ deployment, execute (respectively):: .. _Bandit: https://github.com/openstack/bandit +Database Model Updates +---------------------- + +Deckhand utilizes `Alembic`_ to handle database setup and upgrades. Alembic +provides a straightforward way to manage the migrations necessary from one +database structure version to another through the use of scripts found in +deckhand/alembic/versions. + +Setting up a migration can be automatic or manual. The `Alembic`_ documentation +provides instructions for how to create a new migration. + +Creating automatic migrations requires that the Deckhand database model is +updated in the source code first. With that database model in the code, and +pointing to an existing Deckhand database structure, Alembic can produce the +steps necessary to move from the current version to the next version. + +One way of creating an automatic migration is to deploy a development Deckhand +database using the pre-updated data model and following the following steps:: + + Navigate to the root Deckhand directory + $ export DH_ROOT=$(pwd) + $ mkdir ${DH_ROOT}/alembic_tmp + + Create a deckhand.conf file that will have the correct DB connection string. + $ tox -e genconfig + $ cp ${DH_ROOT}/etc/deckhand/deckhand.conf.sample ${DH_ROOT}/alembic_tmp/deckhand.conf + + Update the connection string to the deckhand db instance e.g.:: + + [Database] + connection = postgresql+psycopg2://deckhand:password@postgresql.ucp.svc.cluster.local:5432/deckhand + + $ export DECKHAND_CONFIG_DIR=${DH_ROOT}/alembic_tmp + $ alembic revision --autogenerate -m "The short description for this change" + + $ rm -r ${DH_ROOT}/alembic_tmp + +This will create a new .py file in the deckhand/alembic/versions directory that +can then be modified to indicate exact steps. The generated migration should +always be inspected to ensure correctness. + +Migrations exist in a linked list of files (the files in versions). Each file +is updated by Alembic to reference its revision linkage. E.g.:: + + # revision identifiers, used by Alembic. + revision = '918bbfd28185' + down_revision = None + branch_labels = None + depends_on = None + +Any manual changes to this linkage must be approached carefully or Alembic will +fail to operate. + +.. _Alembic: http://alembic.zzzcomputing.com/en/latest/ + Troubleshooting --------------- diff --git a/docs/source/glossary.rst b/docs/source/glossary.rst index 021366f9..a5c3dbaf 100644 --- a/docs/source/glossary.rst +++ b/docs/source/glossary.rst @@ -17,6 +17,15 @@ Glossary ======== +A +~ + +.. glossary:: + + Alembic + + Database migration software for Python and SQLAlchemy based databases. + B ~ @@ -55,6 +64,25 @@ K generation system capable of providing key management for services wishing to enable encryption features. +M +~ + +.. glossary:: + + migration (databse) + + A transformation of a databse from one version or structure to another. + Migrations for Deckhand's database are performed using Alembic. + +S +~ + +.. glossary:: + + SQLAlchemy + + Databse toolkit for Python. + U ~ diff --git a/entrypoint.sh b/entrypoint.sh index f0a9cea0..a39d73d1 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -34,18 +34,25 @@ DECKHAND_API_THREADS=${DECKHAND_API_THREADS:-"4"} # The Deckhand configuration directory containing deckhand.conf DECKHAND_CONFIG_DIR=${DECKHAND_CONFIG_DIR:-"/etc/deckhand/deckhand.conf"} +echo "Command: $1 with arguments $@" # Start deckhand application -exec uwsgi \ - -b 32768 \ - --callable deckhand_callable \ - --die-on-term \ - --enable-threads \ - --http :${PORT} \ - --http-timeout $DECKHAND_API_TIMEOUT \ - -L \ - --lazy-apps \ - --master \ - --pyargv "--config-file ${DECKHAND_CONFIG_DIR}/deckhand.conf" \ - --threads $DECKHAND_API_THREADS \ - --workers $DECKHAND_API_WORKERS \ - -w deckhand.cmd +if [ "$1" = 'server' ]; then + exec uwsgi \ + -b 32768 \ + --callable deckhand_callable \ + --die-on-term \ + --enable-threads \ + --http :${PORT} \ + --http-timeout $DECKHAND_API_TIMEOUT \ + -L \ + --lazy-apps \ + --master \ + --pyargv "--config-file ${DECKHAND_CONFIG_DIR}/deckhand.conf" \ + --threads $DECKHAND_API_THREADS \ + --workers $DECKHAND_API_WORKERS \ + -w deckhand.cmd +elif [ "$1" = 'alembic' ]; then + exec alembic ${@:2} +else + echo "Valid commands are 'alembic ' and 'server'" +fi diff --git a/images/deckhand/Dockerfile b/images/deckhand/Dockerfile index c0fb0b6b..1198756b 100644 --- a/images/deckhand/Dockerfile +++ b/images/deckhand/Dockerfile @@ -71,3 +71,5 @@ USER deckhand # Execute entrypoint ENTRYPOINT ["/home/deckhand/entrypoint.sh"] + +CMD ["server"] diff --git a/requirements.txt b/requirements.txt index f0ddf45f..5a976b8b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ # Hacking already pins down pep8, pyflakes and flake8 hacking>=1.0.0 # Apache-2.0 +alembic==0.8.2 # MIT falcon>=1.4.1 # Apache-2.0 pbr!=2.1.0,>=3.1.1 # Apache-2.0 PasteDeploy>=1.5.2 # MIT diff --git a/tools/functional-tests.sh b/tools/functional-tests.sh index 2d7c7be6..08044917 100755 --- a/tools/functional-tests.sh +++ b/tools/functional-tests.sh @@ -202,6 +202,8 @@ ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" if [ -z "$DECKHAND_IMAGE" ]; then log_section "Running Deckhand via uwsgi" + + alembic upgrade head # NOTE(fmontei): Deckhand's database is not configured to work with # multiprocessing. Currently there is a data race on acquiring shared # SQLAlchemy engine pooled connection strings when workers > 1. As a @@ -213,6 +215,11 @@ if [ -z "$DECKHAND_IMAGE" ]; then sleep 5 else log_section "Running Deckhand via Docker" + sudo docker run \ + --rm \ + --net=host \ + -v $CONF_DIR:/etc/deckhand \ + $DECKHAND_IMAGE alembic upgrade head &> $STDOUT sudo docker run \ --rm \ --net=host \ diff --git a/tox.ini b/tox.ini index c2bfc7eb..373b8b78 100644 --- a/tox.ini +++ b/tox.ini @@ -90,7 +90,7 @@ commands = flake8 {posargs} # [H904] Delay string interpolations at logging calls. enable-extensions = H106,H203,H204,H205,H210,H904 ignore = E127,E128,E129,E131,H405 -exclude = .venv,.git,.tox,dist,*lib/python*,*egg,build,releasenotes,docs +exclude = .venv,.git,.tox,dist,*lib/python*,*egg,build,releasenotes,docs,alembic/versions [testenv:docs] deps = -r{toxinidir}/docs/requirements.txt