diff --git a/alembic/versions/51b92375e5c4_initial_shipyard_base.py b/alembic/versions/51b92375e5c4_initial_shipyard_base.py index d21a1a4a..9b3812d4 100644 --- a/alembic/versions/51b92375e5c4_initial_shipyard_base.py +++ b/alembic/versions/51b92375e5c4_initial_shipyard_base.py @@ -73,6 +73,7 @@ def upgrade(): server_default=func.now()), ) + def downgrade(): """ Remove the database objects created by this revision diff --git a/shipyard_airflow/conf/config.py b/shipyard_airflow/conf/config.py index 5cd98c67..446f2711 100644 --- a/shipyard_airflow/conf/config.py +++ b/shipyard_airflow/conf/config.py @@ -214,6 +214,7 @@ SECTIONS = [ ), ] + def register_opts(conf): """ Registers all the sections in this module. diff --git a/shipyard_airflow/conf/opts.py b/shipyard_airflow/conf/opts.py index f02f8e71..c395d4d5 100644 --- a/shipyard_airflow/conf/opts.py +++ b/shipyard_airflow/conf/opts.py @@ -26,6 +26,7 @@ class ConfigSection(object): """ Defines a configuration section """ + def __init__(self, name, title, options, help=None): self.name = name self.title = title diff --git a/shipyard_airflow/control/action_helper.py b/shipyard_airflow/control/action_helper.py index 09855d8a..2c2b573e 100644 --- a/shipyard_airflow/control/action_helper.py +++ b/shipyard_airflow/control/action_helper.py @@ -30,6 +30,7 @@ DAG_STATE_MAPPING = { 'PAUSED': 'Paused' } + def determine_lifecycle(dag_status=None): """ Convert a dag_status to an action_lifecycle value @@ -38,6 +39,7 @@ def determine_lifecycle(dag_status=None): dag_status = 'NONE' return DAG_STATE_MAPPING.get(dag_status.upper()) + def format_action_steps(action_id, steps): """ Converts a list of action step database records to desired format @@ -51,6 +53,7 @@ def format_action_steps(action_id, steps): index=idx + 1)) return steps_response + def format_step(action_id, step, index): """ reformat a step (dictionary) into a common response format diff --git a/shipyard_airflow/control/api.py b/shipyard_airflow/control/api.py index 25a37b5a..4ac5e8da 100644 --- a/shipyard_airflow/control/api.py +++ b/shipyard_airflow/control/api.py @@ -77,10 +77,12 @@ def start_api(): return control_api + class VersionsResource(BaseResource): """ Lists the versions supported by this API """ + def on_get(self, req, resp): resp.body = self.to_json({ 'v1.0': { diff --git a/shipyard_airflow/control/base.py b/shipyard_airflow/control/base.py index 697f3511..34205593 100644 --- a/shipyard_airflow/control/base.py +++ b/shipyard_airflow/control/base.py @@ -128,10 +128,12 @@ class BaseResource(object): """ self.log_message(ctx, logging.ERROR, msg) + class ShipyardRequestContext(object): """ Context object for shipyard resource requests """ + def __init__(self): self.log_level = 'error' self.user = None diff --git a/shipyard_airflow/dags/deploy_site.py b/shipyard_airflow/dags/deploy_site.py index 1b1909a1..2fe26169 100644 --- a/shipyard_airflow/dags/deploy_site.py +++ b/shipyard_airflow/dags/deploy_site.py @@ -54,6 +54,8 @@ Define push function to store the content of 'action' that is defined via 'dag_run' in XCOM so that it can be used by the Operators """ + + def xcom_push(**kwargs): # Pushes action XCom kwargs['ti'].xcom_push(key='action', diff --git a/shipyard_airflow/dags/drydock_deploy_site.py b/shipyard_airflow/dags/drydock_deploy_site.py index 5df71542..2e74025d 100644 --- a/shipyard_airflow/dags/drydock_deploy_site.py +++ b/shipyard_airflow/dags/drydock_deploy_site.py @@ -35,6 +35,7 @@ promenade_conf = config.get('drydock', 'prom_yaml') parent_dag = 'deploy_site' child_dag = 'deploy_site.drydock_build' + def create_drydock_client(parent_dag_name, child_dag_name, args): ''' Create Drydock Client @@ -56,6 +57,7 @@ def create_drydock_client(parent_dag_name, child_dag_name, args): return dag + def drydock_get_design_id(parent_dag_name, child_dag_name, args): ''' Get Design ID @@ -73,6 +75,7 @@ def drydock_get_design_id(parent_dag_name, child_dag_name, args): return dag + def drydock_load_parts(parent_dag_name, child_dag_name, args): ''' Load DryDock Yaml @@ -91,6 +94,7 @@ def drydock_load_parts(parent_dag_name, child_dag_name, args): return dag + def promenade_load_parts(parent_dag_name, child_dag_name, args): ''' Load Promenade Yaml @@ -109,6 +113,7 @@ def promenade_load_parts(parent_dag_name, child_dag_name, args): return dag + def drydock_verify_site(parent_dag_name, child_dag_name, args): ''' Verify connectivity between DryDock and MAAS @@ -126,6 +131,7 @@ def drydock_verify_site(parent_dag_name, child_dag_name, args): return dag + def drydock_prepare_site(parent_dag_name, child_dag_name, args): ''' Prepare site for deployment @@ -143,6 +149,7 @@ def drydock_prepare_site(parent_dag_name, child_dag_name, args): return dag + def drydock_prepare_node(parent_dag_name, child_dag_name, args): ''' Prepare nodes for deployment @@ -160,6 +167,7 @@ def drydock_prepare_node(parent_dag_name, child_dag_name, args): return dag + def drydock_deploy_node(parent_dag_name, child_dag_name, args): ''' Deploy Nodes @@ -188,6 +196,7 @@ DRYDOCK_PREPARE_SITE_DAG_NAME = 'drydock_prepare_site' DRYDOCK_PREPARE_NODE_DAG_NAME = 'drydock_prepare_node' DRYDOCK_DEPLOY_NODE_DAG_NAME = 'drydock_deploy_node' + def deploy_site_drydock(parent_dag_name, child_dag_name, args): ''' Puts all of the drydock deploy site into atomic unit diff --git a/shipyard_airflow/dags/drydock_operator_child.py b/shipyard_airflow/dags/drydock_operator_child.py index 6a5d5053..f3934dd0 100644 --- a/shipyard_airflow/dags/drydock_operator_child.py +++ b/shipyard_airflow/dags/drydock_operator_child.py @@ -18,6 +18,7 @@ import configparser from airflow import DAG from airflow.operators import DryDockOperator + def sub_dag(parent_dag_name, child_dag_name, args, schedule_interval): dag = DAG( '%s.%s' % (parent_dag_name, child_dag_name), diff --git a/shipyard_airflow/dags/drydock_operator_parent.py b/shipyard_airflow/dags/drydock_operator_parent.py index 8cce87e5..a8e0ccfb 100644 --- a/shipyard_airflow/dags/drydock_operator_parent.py +++ b/shipyard_airflow/dags/drydock_operator_parent.py @@ -44,6 +44,8 @@ main_dag = DAG( # Define push function to store the content of 'action' that is # defined via 'dag_run' in XCOM so that it can be used by the # DryDock Operators + + def push(**kwargs): # Pushes action XCom kwargs['ti'].xcom_push(key='action', diff --git a/shipyard_airflow/dags/preflight_checks.py b/shipyard_airflow/dags/preflight_checks.py index cc077594..198fadd4 100644 --- a/shipyard_airflow/dags/preflight_checks.py +++ b/shipyard_airflow/dags/preflight_checks.py @@ -24,6 +24,7 @@ config_path = '/usr/local/airflow/plugins/shipyard.conf' # Note that the shipyard.conf file needs to be placed on a volume # that can be accessed by the containers + def k8s_preflight_check(parent_dag_name, child_dag_name, args): ''' The k8s_preflight_check checks that k8s is in a good state for diff --git a/shipyard_airflow/dags/redeploy_server.py b/shipyard_airflow/dags/redeploy_server.py index 6b06109a..a59ff191 100644 --- a/shipyard_airflow/dags/redeploy_server.py +++ b/shipyard_airflow/dags/redeploy_server.py @@ -52,6 +52,8 @@ Define push function to store the content of 'action' that is defined via 'dag_run' in XCOM so that it can be used by the Operators """ + + def xcom_push(**kwargs): # Pushes action XCom kwargs['ti'].xcom_push(key='action', diff --git a/shipyard_airflow/dags/update_site.py b/shipyard_airflow/dags/update_site.py index bc4e2ca4..595aed82 100644 --- a/shipyard_airflow/dags/update_site.py +++ b/shipyard_airflow/dags/update_site.py @@ -54,6 +54,8 @@ Define push function to store the content of 'action' that is defined via 'dag_run' in XCOM so that it can be used by the Operators """ + + def xcom_push(**kwargs): # Pushes action XCom kwargs['ti'].xcom_push(key='action', diff --git a/shipyard_airflow/db/common_db.py b/shipyard_airflow/db/common_db.py index 8238120f..f5194f57 100644 --- a/shipyard_airflow/db/common_db.py +++ b/shipyard_airflow/db/common_db.py @@ -19,6 +19,7 @@ from shipyard_airflow.errors import DatabaseError LOG = logging.getLogger(__name__) + class DbAccess: """ Base class for simple database access diff --git a/shipyard_airflow/db/errors.py b/shipyard_airflow/db/errors.py index ded955e6..e988ac57 100644 --- a/shipyard_airflow/db/errors.py +++ b/shipyard_airflow/db/errors.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + class AirflowStateError(Exception): def __init__(self, message=""): """ diff --git a/shipyard_airflow/db/shipyard_db.py b/shipyard_airflow/db/shipyard_db.py index 35524d43..6b6fa48f 100644 --- a/shipyard_airflow/db/shipyard_db.py +++ b/shipyard_airflow/db/shipyard_db.py @@ -28,6 +28,7 @@ from shipyard_airflow.db.common_db import DbAccess LOG = logging.getLogger(__name__) CONF = cfg.CONF + class ShipyardDbAccess(DbAccess): """ Shipyard database access diff --git a/shipyard_airflow/errors.py b/shipyard_airflow/errors.py index d44c62e2..6be7b07a 100644 --- a/shipyard_airflow/errors.py +++ b/shipyard_airflow/errors.py @@ -70,6 +70,7 @@ def format_resp(req, resp.content_type = 'application/json' resp.status = status_code + def default_error_serializer(req, resp, exception): """ Writes the default error message body, when we don't handle it otherwise @@ -84,6 +85,7 @@ def default_error_serializer(req, resp, exception): error_list=[{'message': exception.description}] ) + def default_exception_handler(ex, req, resp, params): """ Catch-all execption handler for standardized output. @@ -110,6 +112,7 @@ class AppError(Exception): """ Base error containing enough information to make a shipyard formatted error """ + def __init__(self, title='Internal Server Error', description=None, @@ -146,6 +149,7 @@ class AirflowError(AppError): """ An error to handle errors returned by the Airflow API """ + def __init__(self, description=None, error_list=None): super().__init__( title='Error response from Airflow', @@ -155,10 +159,12 @@ class AirflowError(AppError): retry=False ) + class DatabaseError(AppError): """ An error to handle general api errors. """ + def __init__(self, description=None, error_list=None, @@ -178,6 +184,7 @@ class ApiError(AppError): """ An error to handle general api errors. """ + def __init__(self, description="", error_list=None, @@ -197,6 +204,7 @@ class InvalidFormatError(AppError): """ An exception to cover invalid input formatting """ + def __init__(self, title, description="Not Specified", error_list=None): super().__init__( diff --git a/shipyard_airflow/plugins/drydock_operators.py b/shipyard_airflow/plugins/drydock_operators.py index 75c14ce0..4519144b 100644 --- a/shipyard_airflow/plugins/drydock_operators.py +++ b/shipyard_airflow/plugins/drydock_operators.py @@ -403,32 +403,33 @@ class DryDockOperator(BaseOperator): def drydock_query_task(self, drydock_client, interval, time_out, task_id, desired_state): - # Calculate number of times to execute the 'for' loop - end_range = int(time_out / interval) + # Calculate number of times to execute the 'for' loop + end_range = int(time_out / interval) - # Query task state - for i in range(0, end_range + 1): + # Query task state + for i in range(0, end_range + 1): - # Retrieve current task state - task_state = drydock_client.get_task(self.task_id) - logging.info(task_state) + # Retrieve current task state + task_state = drydock_client.get_task(self.task_id) + logging.info(task_state) - # Return Time Out Exception - if task_state['status'] == 'running' and i == end_range: - logging.info('Timed Out!') - return 'timed_out' + # Return Time Out Exception + if task_state['status'] == 'running' and i == end_range: + logging.info('Timed Out!') + return 'timed_out' - # Exit 'for' loop if task is in 'complete' state - if task_state['status'] == 'complete': - break - else: - time.sleep(interval) - - # Get final task state - if task_state['result'] == desired_state: - return drydock_client.get_task(self.task_id) + # Exit 'for' loop if task is in 'complete' state + if task_state['status'] == 'complete': + break else: - return 'task_failed' + time.sleep(interval) + + # Get final task state + if task_state['result'] == desired_state: + return drydock_client.get_task(self.task_id) + else: + return 'task_failed' + class DryDockClientPlugin(AirflowPlugin): name = "drydock_client_plugin" diff --git a/shipyard_airflow/plugins/rest_api_plugin.py b/shipyard_airflow/plugins/rest_api_plugin.py index 23d40003..4b13484b 100644 --- a/shipyard_airflow/plugins/rest_api_plugin.py +++ b/shipyard_airflow/plugins/rest_api_plugin.py @@ -1,4 +1,4 @@ -# Original Source Code: https://github.com/teamclairvoyant/airflow-rest-api-plugin +# Original Source Code: https://github.com/teamclairvoyant/airflow-rest-api-plugin # Date when airflow-rest-api-plugin source codes were downloaded: 2017-06-27 # Kept majority of the codes and removed those that are not needed for our purpose @@ -780,4 +780,3 @@ class REST_API_Plugin(AirflowPlugin): executors = [] admin_views = [rest_api_view] menu_links = [] - diff --git a/tests/unit/control/__init__.py b/tests/unit/control/__init__.py index 1ff35516..de8f20a1 100644 --- a/tests/unit/control/__init__.py +++ b/tests/unit/control/__init__.py @@ -15,6 +15,7 @@ import pytest from shipyard_airflow.conf import config + @pytest.fixture def setup_config(): """ diff --git a/tests/unit/control/test_actions_api.py b/tests/unit/control/test_actions_api.py index 36144888..0062e3f9 100644 --- a/tests/unit/control/test_actions_api.py +++ b/tests/unit/control/test_actions_api.py @@ -142,6 +142,7 @@ def tasks_db(): }, ] + def airflow_stub(**kwargs): """ asserts that the airflow invocation method was called with the right @@ -152,12 +153,14 @@ def airflow_stub(**kwargs): print(kwargs) return '2017-09-06 14:10:08.528402' + def insert_action_stub(**kwargs): """ asserts that the insert action was called with the right parameters """ assert kwargs['action'] + def audit_control_command_db(action_audit): """ Stub for inserting the invoke record @@ -167,6 +170,7 @@ def audit_control_command_db(action_audit): context = ShipyardRequestContext() + def test_get_all_actions(): """ Tests the main response from get all actions @@ -188,6 +192,7 @@ def test_get_all_actions(): assert len(action['steps']) == 3 assert action['dag_status'] == 'SUCCESS' + def test_create_action(): action_resource = ActionsResource() action_resource.get_all_actions_db = actions_db diff --git a/tests/unit/control/test_actions_control_api.py b/tests/unit/control/test_actions_control_api.py index 157a18c3..3b77e267 100644 --- a/tests/unit/control/test_actions_control_api.py +++ b/tests/unit/control/test_actions_control_api.py @@ -47,6 +47,7 @@ def actions_db(action_id): 'context_marker': '8-4-4-4-12a' } + def control_dag_run(dag_id, execution_date, expected_state, @@ -56,9 +57,11 @@ def control_dag_run(dag_id, else: pass + def audit_control_command_db(action_audit): pass + def test_get_action(): """ Tests the main response from get all actions diff --git a/tests/unit/control/test_actions_id_api.py b/tests/unit/control/test_actions_id_api.py index 16175c7d..a337476d 100644 --- a/tests/unit/control/test_actions_id_api.py +++ b/tests/unit/control/test_actions_id_api.py @@ -20,6 +20,7 @@ DATE_TWO = datetime(2017, 9, 13, 11, 13, 5, 57000) DATE_ONE_STR = DATE_ONE.strftime('%Y-%m-%dT%H:%M:%S') DATE_TWO_STR = DATE_TWO.strftime('%Y-%m-%dT%H:%M:%S') + def actions_db(action_id): """ replaces the actual db call @@ -35,6 +36,7 @@ def actions_db(action_id): 'context_marker': '8-4-4-4-12a' } + def dag_runs_db(dag_id, execution_date): """ replaces the actual db call @@ -49,6 +51,7 @@ def dag_runs_db(dag_id, execution_date): 'end_date': DATE_ONE }] + def tasks_db(dag_id, execution_date): """ replaces the actual db call @@ -98,6 +101,7 @@ def tasks_db(dag_id, execution_date): } ] + def get_validations(action_id): """ Stub to return validations @@ -111,6 +115,7 @@ def get_validations(action_id): } ] + def get_ac_audit(action_id): """ Stub to return command audit response @@ -132,6 +137,7 @@ def get_ac_audit(action_id): } ] + def test_get_action(): """ Tests the main response from get all actions diff --git a/tests/unit/control/test_actions_steps_id_api.py b/tests/unit/control/test_actions_steps_id_api.py index ae94d408..be6d2f81 100644 --- a/tests/unit/control/test_actions_steps_id_api.py +++ b/tests/unit/control/test_actions_steps_id_api.py @@ -38,6 +38,7 @@ def actions_db(action_id): 'context_marker': '8-4-4-4-12a' } + def tasks_db(dag_id, execution_date): """ replaces the actual db call @@ -87,6 +88,7 @@ def tasks_db(dag_id, execution_date): } ] + def test_get_action_steps(): """ Tests the main response from get all actions diff --git a/tests/unit/control/test_actions_validations_id_api.py b/tests/unit/control/test_actions_validations_id_api.py index 527ce7ad..dba5cbef 100644 --- a/tests/unit/control/test_actions_validations_id_api.py +++ b/tests/unit/control/test_actions_validations_id_api.py @@ -17,6 +17,7 @@ from shipyard_airflow.control.actions_validations_id_api import ( ) from shipyard_airflow.errors import ApiError + def actions_db(action_id): """ replaces the actual db call @@ -35,6 +36,7 @@ def actions_db(action_id): 'context_marker': '8-4-4-4-12a' } + def get_validations(validation_id): """ Stub to return validations @@ -49,6 +51,7 @@ def get_validations(validation_id): else: return None + def test_get_action_validation(): """ Tests the main response from get all actions diff --git a/tests/unit/plugins/test_concurrency_check_operator.py b/tests/unit/plugins/test_concurrency_check_operator.py index 5b4fb87c..bf6050ef 100644 --- a/tests/unit/plugins/test_concurrency_check_operator.py +++ b/tests/unit/plugins/test_concurrency_check_operator.py @@ -98,6 +98,7 @@ def test_find_conflicting_dag(): assert cco.find_conflicting_dag('buffalo') != 'monkey' assert cco.find_conflicting_dag('buffalo') == 'chicken' + def test_execute_exception(): """ Run the whole execute function for testing @@ -114,6 +115,7 @@ def test_execute_exception(): except AirflowException as airflow_exception: assert 'Aborting run' in airflow_exception.args[0] + def test_execute_success(): """ Run the whole execute function for testing - successfully! diff --git a/tox.ini b/tox.ini index 2327bad2..9d59c6e5 100644 --- a/tox.ini +++ b/tox.ini @@ -28,7 +28,7 @@ commands = oslopolicy-sample-generator --config-file=generator/policy-generator. # NOTE(Bryan Strassner) ignoring F841 because of the airflow example pattern # of naming variables even if they aren't used for DAGs and Operators. # Doing so adds readability and context in this case. -ignore = E302,H306,D100,D101,D102,F841 +ignore = F841 # NOTE(Bryan Strassner) excluding 3rd party and generated code that is brought into the # codebase. exclude = *plugins/rest_api_plugin.py,*lib/python*,*egg,.git*,*.md,.tox*,alembic/env.py,build/*