38e58cfd30
This change introduces a large section of the API for the next major version of Shipyard - the action api. By interfacing with Airflow, Shipyard will invoke workflows and allow for controlling and querying status of those workflows. Foundationally, this patchset introduces a lot of framework code for other apis, including error handling to a common output format, database interaction for persistence of action information, and use of oslo_config for configuration support. Add GET all actions primary code - db connection not yet impl Update base classes to have more structure Add POST actions framework Add GET action by id Add GET of validations and steps Add control api Add unit tests of action api methods Re-Removed duplicate deps from test reqs Add routes for API Removed a lot of code better handled by falcon directly Cleaned up error flows- handlers and defaults Refactored existing airflow tests to match standard output format Updated json validation to be more specific Added basic start for alembic Added alembic upgrade at startup Added table creation definitions Added base revision for alembic upgrade Bug fixes - DB queries, airflow comm, logic issues, logging issues Bug fixes - date formats and alignment of keys between systems Exclusions to bandit / tox.ini Resolved merge conflicts with integration of auth Update to use oslo config and PBR Update the context middleware to check uuid in a less contentious way Removed routes and resources for regions endpoint - not used Add auth policies for action api Restructure execptions to be consistent class hierarchy and common handler Add generation of config and policy examples Update tests to init configs Update database configs to not use env. vars Removed examples directory, it was no longer accurate Addressed/removed several TODOs - left some behind as well Aligned input to DAGs with action: header Retrieved all sub-steps for dags Expanded step information Refactored auth handling for better logging rename create_actions policy to create_action removed some templated file comments in env.py generated by alembic updated inconsistent exception parameters updated to use ulid instead of uuid for action ids added action control audit code per review suggestion Fixed correlation date betwen dags/actions by more string parsing Change-Id: I2f9ea5250923f45456aa86826e344fc055bba762
83 lines
2.9 KiB
Python
83 lines
2.9 KiB
Python
"""initial shipyard base
|
|
|
|
Revision ID: 51b92375e5c4
|
|
Revises:
|
|
Create Date: 2017-09-12 11:12:23.768269
|
|
|
|
"""
|
|
from alembic import op
|
|
import sqlalchemy as sa
|
|
from sqlalchemy import (types, func)
|
|
from sqlalchemy.dialects import postgresql as pg
|
|
|
|
|
|
# revision identifiers, used by Alembic.
|
|
revision = '51b92375e5c4'
|
|
down_revision = None
|
|
branch_labels = None
|
|
depends_on = None
|
|
|
|
|
|
def upgrade():
|
|
"""
|
|
Create the initial tables needed by shipyard
|
|
26 character IDs are ULIDs. See: https://github.com/mdipierro/ulid
|
|
"""
|
|
op.create_table(
|
|
'actions',
|
|
# ULID key for the action
|
|
sa.Column('id', types.String(26), primary_key=True),
|
|
# The name of the action invoked
|
|
sa.Column('name', types.String(50), nullable=False),
|
|
# The parameters passed by the user to the action
|
|
sa.Column('parameters', pg.JSONB, nullable=True),
|
|
# The DAG/workflow name used in airflow, if applicable
|
|
sa.Column('dag_id', sa.Text, nullable=True),
|
|
# The DAG/workflow execution time string from airflow, if applicable
|
|
sa.Column('dag_execution_date', sa.Text, nullable=True),
|
|
# The invoking user
|
|
sa.Column('user', sa.Text, nullable=False),
|
|
# Timestamp of when an action was invoked
|
|
sa.Column('datetime',
|
|
types.TIMESTAMP(timezone=True),
|
|
server_default=func.now()),
|
|
# The user provided or shipayrd generated context marker
|
|
sa.Column('context_marker', types.String(36), nullable=False)
|
|
)
|
|
|
|
op.create_table(
|
|
'preflight_validation_failures',
|
|
# ID (ULID) of the preflight validation failure
|
|
sa.Column('id', types.String(26), primary_key=True),
|
|
# The ID of action this failure is associated with
|
|
sa.Column('action_id', types.String(26), nullable=False),
|
|
# The common language name of the validation that failed
|
|
sa.Column('validation_name', sa.Text, nullable=True),
|
|
# The text indicating details of the failure
|
|
sa.Column('details', sa.Text, nullable=True),
|
|
)
|
|
|
|
op.create_table(
|
|
'action_command_audit',
|
|
# ID (ULID) of the audit
|
|
sa.Column('id', types.String(26), primary_key=True),
|
|
# The ID of action this audit record
|
|
sa.Column('action_id', types.String(26), nullable=False),
|
|
# The text indicating command invoked
|
|
sa.Column('command', sa.Text, nullable=False),
|
|
# The user that invoked the command
|
|
sa.Column('user', sa.Text, nullable=False),
|
|
# Timestamp of when the command was invoked
|
|
sa.Column('datetime',
|
|
types.TIMESTAMP(timezone=True),
|
|
server_default=func.now()),
|
|
)
|
|
|
|
def downgrade():
|
|
"""
|
|
Remove the database objects created by this revision
|
|
"""
|
|
op.drop_table('actions')
|
|
op.drop_table('preflight_validation_failures')
|
|
op.drop_table('action_command_audit')
|