Browse Source

Linting: Make Shipyard mostly pep8 compliant

This PS makes shipyard mostly pep8 compliant, with the exeption of:
./shipyard_airflow/plugins/rest_api_plugin.py which will require some
rework to make compliant.

Change-Id: I913f7ac696eb5558c1fed7c4f4104211bf4b5876
changes/87/569187/1
Pete Birley 4 years ago
parent
commit
6ae83ed258
  1. 1
      alembic/versions/51b92375e5c4_initial_shipyard_base.py
  2. 1
      shipyard_airflow/conf/config.py
  3. 1
      shipyard_airflow/conf/opts.py
  4. 3
      shipyard_airflow/control/action_helper.py
  5. 2
      shipyard_airflow/control/api.py
  6. 2
      shipyard_airflow/control/base.py
  7. 2
      shipyard_airflow/dags/deploy_site.py
  8. 9
      shipyard_airflow/dags/drydock_deploy_site.py
  9. 1
      shipyard_airflow/dags/drydock_operator_child.py
  10. 2
      shipyard_airflow/dags/drydock_operator_parent.py
  11. 1
      shipyard_airflow/dags/preflight_checks.py
  12. 2
      shipyard_airflow/dags/redeploy_server.py
  13. 2
      shipyard_airflow/dags/update_site.py
  14. 1
      shipyard_airflow/db/common_db.py
  15. 1
      shipyard_airflow/db/errors.py
  16. 1
      shipyard_airflow/db/shipyard_db.py
  17. 8
      shipyard_airflow/errors.py
  18. 51
      shipyard_airflow/plugins/drydock_operators.py
  19. 3
      shipyard_airflow/plugins/rest_api_plugin.py
  20. 1
      tests/unit/control/__init__.py
  21. 5
      tests/unit/control/test_actions_api.py
  22. 3
      tests/unit/control/test_actions_control_api.py
  23. 6
      tests/unit/control/test_actions_id_api.py
  24. 2
      tests/unit/control/test_actions_steps_id_api.py
  25. 3
      tests/unit/control/test_actions_validations_id_api.py
  26. 2
      tests/unit/plugins/test_concurrency_check_operator.py
  27. 2
      tox.ini

1
alembic/versions/51b92375e5c4_initial_shipyard_base.py

@ -73,6 +73,7 @@ def upgrade():
server_default=func.now()),
)
def downgrade():
"""
Remove the database objects created by this revision

1
shipyard_airflow/conf/config.py

@ -214,6 +214,7 @@ SECTIONS = [
),
]
def register_opts(conf):
"""
Registers all the sections in this module.

1
shipyard_airflow/conf/opts.py

@ -26,6 +26,7 @@ class ConfigSection(object):
"""
Defines a configuration section
"""
def __init__(self, name, title, options, help=None):
self.name = name
self.title = title

3
shipyard_airflow/control/action_helper.py

@ -30,6 +30,7 @@ DAG_STATE_MAPPING = {
'PAUSED': 'Paused'
}
def determine_lifecycle(dag_status=None):
"""
Convert a dag_status to an action_lifecycle value
@ -38,6 +39,7 @@ def determine_lifecycle(dag_status=None):
dag_status = 'NONE'
return DAG_STATE_MAPPING.get(dag_status.upper())
def format_action_steps(action_id, steps):
"""
Converts a list of action step database records to desired format
@ -51,6 +53,7 @@ def format_action_steps(action_id, steps):
index=idx + 1))
return steps_response
def format_step(action_id, step, index):
"""
reformat a step (dictionary) into a common response format

2
shipyard_airflow/control/api.py

@ -77,10 +77,12 @@ def start_api():
return control_api
class VersionsResource(BaseResource):
"""
Lists the versions supported by this API
"""
def on_get(self, req, resp):
resp.body = self.to_json({
'v1.0': {

2
shipyard_airflow/control/base.py

@ -128,10 +128,12 @@ class BaseResource(object):
"""
self.log_message(ctx, logging.ERROR, msg)
class ShipyardRequestContext(object):
"""
Context object for shipyard resource requests
"""
def __init__(self):
self.log_level = 'error'
self.user = None

2
shipyard_airflow/dags/deploy_site.py

@ -54,6 +54,8 @@ Define push function to store the content of 'action' that is
defined via 'dag_run' in XCOM so that it can be used by the
Operators
"""
def xcom_push(**kwargs):
# Pushes action XCom
kwargs['ti'].xcom_push(key='action',

9
shipyard_airflow/dags/drydock_deploy_site.py

@ -35,6 +35,7 @@ promenade_conf = config.get('drydock', 'prom_yaml')
parent_dag = 'deploy_site'
child_dag = 'deploy_site.drydock_build'
def create_drydock_client(parent_dag_name, child_dag_name, args):
'''
Create Drydock Client
@ -56,6 +57,7 @@ def create_drydock_client(parent_dag_name, child_dag_name, args):
return dag
def drydock_get_design_id(parent_dag_name, child_dag_name, args):
'''
Get Design ID
@ -73,6 +75,7 @@ def drydock_get_design_id(parent_dag_name, child_dag_name, args):
return dag
def drydock_load_parts(parent_dag_name, child_dag_name, args):
'''
Load DryDock Yaml
@ -91,6 +94,7 @@ def drydock_load_parts(parent_dag_name, child_dag_name, args):
return dag
def promenade_load_parts(parent_dag_name, child_dag_name, args):
'''
Load Promenade Yaml
@ -109,6 +113,7 @@ def promenade_load_parts(parent_dag_name, child_dag_name, args):
return dag
def drydock_verify_site(parent_dag_name, child_dag_name, args):
'''
Verify connectivity between DryDock and MAAS
@ -126,6 +131,7 @@ def drydock_verify_site(parent_dag_name, child_dag_name, args):
return dag
def drydock_prepare_site(parent_dag_name, child_dag_name, args):
'''
Prepare site for deployment
@ -143,6 +149,7 @@ def drydock_prepare_site(parent_dag_name, child_dag_name, args):
return dag
def drydock_prepare_node(parent_dag_name, child_dag_name, args):
'''
Prepare nodes for deployment
@ -160,6 +167,7 @@ def drydock_prepare_node(parent_dag_name, child_dag_name, args):
return dag
def drydock_deploy_node(parent_dag_name, child_dag_name, args):
'''
Deploy Nodes
@ -188,6 +196,7 @@ DRYDOCK_PREPARE_SITE_DAG_NAME = 'drydock_prepare_site'
DRYDOCK_PREPARE_NODE_DAG_NAME = 'drydock_prepare_node'
DRYDOCK_DEPLOY_NODE_DAG_NAME = 'drydock_deploy_node'
def deploy_site_drydock(parent_dag_name, child_dag_name, args):
'''
Puts all of the drydock deploy site into atomic unit

1
shipyard_airflow/dags/drydock_operator_child.py

@ -18,6 +18,7 @@ import configparser
from airflow import DAG
from airflow.operators import DryDockOperator
def sub_dag(parent_dag_name, child_dag_name, args, schedule_interval):
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),

2
shipyard_airflow/dags/drydock_operator_parent.py

@ -44,6 +44,8 @@ main_dag = DAG(
# Define push function to store the content of 'action' that is
# defined via 'dag_run' in XCOM so that it can be used by the
# DryDock Operators
def push(**kwargs):
# Pushes action XCom
kwargs['ti'].xcom_push(key='action',

1
shipyard_airflow/dags/preflight_checks.py

@ -24,6 +24,7 @@ config_path = '/usr/local/airflow/plugins/shipyard.conf'
# Note that the shipyard.conf file needs to be placed on a volume
# that can be accessed by the containers
def k8s_preflight_check(parent_dag_name, child_dag_name, args):
'''
The k8s_preflight_check checks that k8s is in a good state for

2
shipyard_airflow/dags/redeploy_server.py

@ -52,6 +52,8 @@ Define push function to store the content of 'action' that is
defined via 'dag_run' in XCOM so that it can be used by the
Operators
"""
def xcom_push(**kwargs):
# Pushes action XCom
kwargs['ti'].xcom_push(key='action',

2
shipyard_airflow/dags/update_site.py

@ -54,6 +54,8 @@ Define push function to store the content of 'action' that is
defined via 'dag_run' in XCOM so that it can be used by the
Operators
"""
def xcom_push(**kwargs):
# Pushes action XCom
kwargs['ti'].xcom_push(key='action',

1
shipyard_airflow/db/common_db.py

@ -19,6 +19,7 @@ from shipyard_airflow.errors import DatabaseError
LOG = logging.getLogger(__name__)
class DbAccess:
"""
Base class for simple database access

1
shipyard_airflow/db/errors.py

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
class AirflowStateError(Exception):
def __init__(self, message=""):
"""

1
shipyard_airflow/db/shipyard_db.py

@ -28,6 +28,7 @@ from shipyard_airflow.db.common_db import DbAccess
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class ShipyardDbAccess(DbAccess):
"""
Shipyard database access

8
shipyard_airflow/errors.py

@ -70,6 +70,7 @@ def format_resp(req,
resp.content_type = 'application/json'
resp.status = status_code
def default_error_serializer(req, resp, exception):
"""
Writes the default error message body, when we don't handle it otherwise
@ -84,6 +85,7 @@ def default_error_serializer(req, resp, exception):
error_list=[{'message': exception.description}]
)
def default_exception_handler(ex, req, resp, params):
"""
Catch-all execption handler for standardized output.
@ -110,6 +112,7 @@ class AppError(Exception):
"""
Base error containing enough information to make a shipyard formatted error
"""
def __init__(self,
title='Internal Server Error',
description=None,
@ -146,6 +149,7 @@ class AirflowError(AppError):
"""
An error to handle errors returned by the Airflow API
"""
def __init__(self, description=None, error_list=None):
super().__init__(
title='Error response from Airflow',
@ -155,10 +159,12 @@ class AirflowError(AppError):
retry=False
)
class DatabaseError(AppError):
"""
An error to handle general api errors.
"""
def __init__(self,
description=None,
error_list=None,
@ -178,6 +184,7 @@ class ApiError(AppError):
"""
An error to handle general api errors.
"""
def __init__(self,
description="",
error_list=None,
@ -197,6 +204,7 @@ class InvalidFormatError(AppError):
"""
An exception to cover invalid input formatting
"""
def __init__(self, title, description="Not Specified", error_list=None):
super().__init__(

51
shipyard_airflow/plugins/drydock_operators.py

@ -403,32 +403,33 @@ class DryDockOperator(BaseOperator):
def drydock_query_task(self, drydock_client, interval, time_out,
task_id, desired_state):
# Calculate number of times to execute the 'for' loop
end_range = int(time_out / interval)
# Query task state
for i in range(0, end_range + 1):
# Retrieve current task state
task_state = drydock_client.get_task(self.task_id)
logging.info(task_state)
# Return Time Out Exception
if task_state['status'] == 'running' and i == end_range:
logging.info('Timed Out!')
return 'timed_out'
# Exit 'for' loop if task is in 'complete' state
if task_state['status'] == 'complete':
break
else:
time.sleep(interval)
# Get final task state
if task_state['result'] == desired_state:
return drydock_client.get_task(self.task_id)
# Calculate number of times to execute the 'for' loop
end_range = int(time_out / interval)
# Query task state
for i in range(0, end_range + 1):
# Retrieve current task state
task_state = drydock_client.get_task(self.task_id)
logging.info(task_state)
# Return Time Out Exception
if task_state['status'] == 'running' and i == end_range:
logging.info('Timed Out!')
return 'timed_out'
# Exit 'for' loop if task is in 'complete' state
if task_state['status'] == 'complete':
break
else:
return 'task_failed'
time.sleep(interval)
# Get final task state
if task_state['result'] == desired_state:
return drydock_client.get_task(self.task_id)
else:
return 'task_failed'
class DryDockClientPlugin(AirflowPlugin):
name = "drydock_client_plugin"

3
shipyard_airflow/plugins/rest_api_plugin.py

@ -1,4 +1,4 @@
# Original Source Code: https://github.com/teamclairvoyant/airflow-rest-api-plugin
# Original Source Code: https://github.com/teamclairvoyant/airflow-rest-api-plugin
# Date when airflow-rest-api-plugin source codes were downloaded: 2017-06-27
# Kept majority of the codes and removed those that are not needed for our purpose
@ -780,4 +780,3 @@ class REST_API_Plugin(AirflowPlugin):
executors = []
admin_views = [rest_api_view]
menu_links = []

1
tests/unit/control/__init__.py

@ -15,6 +15,7 @@ import pytest
from shipyard_airflow.conf import config
@pytest.fixture
def setup_config():
"""

5
tests/unit/control/test_actions_api.py

@ -142,6 +142,7 @@ def tasks_db():
},
]
def airflow_stub(**kwargs):
"""
asserts that the airflow invocation method was called with the right
@ -152,12 +153,14 @@ def airflow_stub(**kwargs):
print(kwargs)
return '2017-09-06 14:10:08.528402'
def insert_action_stub(**kwargs):
"""
asserts that the insert action was called with the right parameters
"""
assert kwargs['action']
def audit_control_command_db(action_audit):
"""
Stub for inserting the invoke record
@ -167,6 +170,7 @@ def audit_control_command_db(action_audit):
context = ShipyardRequestContext()
def test_get_all_actions():
"""
Tests the main response from get all actions
@ -188,6 +192,7 @@ def test_get_all_actions():
assert len(action['steps']) == 3
assert action['dag_status'] == 'SUCCESS'
def test_create_action():
action_resource = ActionsResource()
action_resource.get_all_actions_db = actions_db

3
tests/unit/control/test_actions_control_api.py

@ -47,6 +47,7 @@ def actions_db(action_id):
'context_marker': '8-4-4-4-12a'
}
def control_dag_run(dag_id,
execution_date,
expected_state,
@ -56,9 +57,11 @@ def control_dag_run(dag_id,
else:
pass
def audit_control_command_db(action_audit):
pass
def test_get_action():
"""
Tests the main response from get all actions

6
tests/unit/control/test_actions_id_api.py

@ -20,6 +20,7 @@ DATE_TWO = datetime(2017, 9, 13, 11, 13, 5, 57000)
DATE_ONE_STR = DATE_ONE.strftime('%Y-%m-%dT%H:%M:%S')
DATE_TWO_STR = DATE_TWO.strftime('%Y-%m-%dT%H:%M:%S')
def actions_db(action_id):
"""
replaces the actual db call
@ -35,6 +36,7 @@ def actions_db(action_id):
'context_marker': '8-4-4-4-12a'
}
def dag_runs_db(dag_id, execution_date):
"""
replaces the actual db call
@ -49,6 +51,7 @@ def dag_runs_db(dag_id, execution_date):
'end_date': DATE_ONE
}]
def tasks_db(dag_id, execution_date):
"""
replaces the actual db call
@ -98,6 +101,7 @@ def tasks_db(dag_id, execution_date):
}
]
def get_validations(action_id):
"""
Stub to return validations
@ -111,6 +115,7 @@ def get_validations(action_id):
}
]
def get_ac_audit(action_id):
"""
Stub to return command audit response
@ -132,6 +137,7 @@ def get_ac_audit(action_id):
}
]
def test_get_action():
"""
Tests the main response from get all actions

2
tests/unit/control/test_actions_steps_id_api.py

@ -38,6 +38,7 @@ def actions_db(action_id):
'context_marker': '8-4-4-4-12a'
}
def tasks_db(dag_id, execution_date):
"""
replaces the actual db call
@ -87,6 +88,7 @@ def tasks_db(dag_id, execution_date):
}
]
def test_get_action_steps():
"""
Tests the main response from get all actions

3
tests/unit/control/test_actions_validations_id_api.py

@ -17,6 +17,7 @@ from shipyard_airflow.control.actions_validations_id_api import (
)
from shipyard_airflow.errors import ApiError
def actions_db(action_id):
"""
replaces the actual db call
@ -35,6 +36,7 @@ def actions_db(action_id):
'context_marker': '8-4-4-4-12a'
}
def get_validations(validation_id):
"""
Stub to return validations
@ -49,6 +51,7 @@ def get_validations(validation_id):
else:
return None
def test_get_action_validation():
"""
Tests the main response from get all actions

2
tests/unit/plugins/test_concurrency_check_operator.py

@ -98,6 +98,7 @@ def test_find_conflicting_dag():
assert cco.find_conflicting_dag('buffalo') != 'monkey'
assert cco.find_conflicting_dag('buffalo') == 'chicken'
def test_execute_exception():
"""
Run the whole execute function for testing
@ -114,6 +115,7 @@ def test_execute_exception():
except AirflowException as airflow_exception:
assert 'Aborting run' in airflow_exception.args[0]
def test_execute_success():
"""
Run the whole execute function for testing - successfully!

2
tox.ini

@ -28,7 +28,7 @@ commands = oslopolicy-sample-generator --config-file=generator/policy-generator.
# NOTE(Bryan Strassner) ignoring F841 because of the airflow example pattern
# of naming variables even if they aren't used for DAGs and Operators.
# Doing so adds readability and context in this case.
ignore = E302,H306,D100,D101,D102,F841
ignore = F841
# NOTE(Bryan Strassner) excluding 3rd party and generated code that is brought into the
# codebase.
exclude = *plugins/rest_api_plugin.py,*lib/python*,*egg,.git*,*.md,.tox*,alembic/env.py,build/*
Loading…
Cancel
Save