222 lines
7.5 KiB
Python
222 lines
7.5 KiB
Python
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
from datetime import datetime
|
|
from unittest import mock
|
|
from unittest.mock import patch
|
|
|
|
import falcon
|
|
import pytest
|
|
import requests
|
|
|
|
from shipyard_airflow.control.action.actions_steps_id_logs_api import \
|
|
ActionsStepsLogsResource
|
|
from shipyard_airflow.errors import ApiError
|
|
from tests.unit.control import common
|
|
|
|
# Define Global Variables
|
|
DATE_ONE = datetime(2018, 4, 5, 16, 29, 11)
|
|
DATE_TWO = datetime(2018, 4, 5, 16, 29, 17, 630472)
|
|
DATE_THREE = datetime(2018, 4, 5, 16, 29, 21, 984263)
|
|
DATE_FOUR = datetime(2018, 4, 5, 16, 29, 13, 698703)
|
|
DATE_ONE_STR = DATE_ONE.strftime('%Y-%m-%d %H:%M:%S')
|
|
DATE_TWO_STR = DATE_TWO.strftime('%Y-%m-%d %H:%M:%S.%f')
|
|
DATE_THREE_STR = DATE_THREE.strftime('%Y-%m-%d %H:%M:%S.%f')
|
|
DATE_FOUR_STR = DATE_FOUR.strftime('%Y-%m-%d %H:%M:%S.%f')
|
|
|
|
EXECUTION_DATE_STR = DATE_ONE.strftime('%Y-%m-%dT%H:%M:%S')
|
|
|
|
KWARGS = {
|
|
'action_id': '01C9VVQSCFS7V9QB5GBS3WFVSE',
|
|
'step_id': 'action_xcom',
|
|
'try_number': 2
|
|
}
|
|
|
|
ACTIONS_DB = {
|
|
'id': '01C9VVQSCFS7V9QB5GBS3WFVSE',
|
|
'name': 'deploy_site',
|
|
'parameters': {},
|
|
'dag_id': 'deploy_site',
|
|
'dag_execution_date': DATE_ONE_STR,
|
|
'user': 'shipyard',
|
|
'timestamp': DATE_ONE_STR,
|
|
'context_marker': '00c6d78e-fb18-4f55-8a1a-b659a516850d'
|
|
}
|
|
|
|
TASK_INSTANCE_DB = [
|
|
{
|
|
'task_id': 'action_xcom',
|
|
'dag_id': 'deploy_site',
|
|
'execution_date': DATE_ONE,
|
|
'start_date': DATE_TWO,
|
|
'end_date': DATE_THREE,
|
|
'duration': '4.353791',
|
|
'state': 'success',
|
|
'try_number': '1',
|
|
'hostname': 'airflow-worker-0.ucp.svc.cluster.local',
|
|
'unixname': 'airflow',
|
|
'job_id': '2',
|
|
'pool': 'default',
|
|
'priority_weight': '8',
|
|
'operator': 'PythonOperator',
|
|
'queued_dttm': DATE_FOUR,
|
|
'pid': '290',
|
|
'max_tries': '0'
|
|
}, {
|
|
'task_id': 'dag_concurrency_check',
|
|
'dag_id': 'deploy_site',
|
|
'execution_date': DATE_ONE,
|
|
'start_date': DATE_TWO,
|
|
'end_date': DATE_THREE,
|
|
'duration': '4.034112',
|
|
'state': 'success',
|
|
'try_number': '1',
|
|
'hostname': 'airflow-worker-1.ucp.svc.cluster.local',
|
|
'unixname': 'airflow',
|
|
'job_id': '3',
|
|
'pool': 'default',
|
|
'priority_weight': '7',
|
|
'operator': 'ConcurrencyCheckOperator',
|
|
'queued_dttm': DATE_FOUR,
|
|
'pid': '324',
|
|
'max_tries': '0'
|
|
}, {
|
|
'task_id': 'k8s_preflight_check',
|
|
'dag_id': 'deploy_site',
|
|
'execution_date': DATE_ONE,
|
|
'start_date': DATE_TWO,
|
|
'end_date': DATE_THREE,
|
|
'duration': '4.452571',
|
|
'state': 'failed',
|
|
'try_number': '1',
|
|
'hostname': 'airflow-worker-0.ucp.svc.cluster.local',
|
|
'unixname': 'airflow',
|
|
'job_id': '7',
|
|
'pool': 'default',
|
|
'priority_weight': '1',
|
|
'operator': 'K8sHealthCheckOperator',
|
|
'queued_dttm': DATE_FOUR,
|
|
'pid': '394',
|
|
'max_tries': '0'
|
|
}
|
|
]
|
|
|
|
XCOM_RUN_LOGS = """
|
|
Running on host airflow-worker-0.airflow-worker-discovery.ucp.svc.cluster.local
|
|
Dependencies all met for TaskInstance: deploy_site.action_xcom
|
|
Dependencies all met for TaskInstance: deploy_site.action_xcom
|
|
INFO -
|
|
--------------------------------------------------------------------------------
|
|
Starting attempt 1 of 1
|
|
--------------------------------------------------------------------------------
|
|
|
|
Executing Task(PythonOperator): action_xcom
|
|
Running: ['bash', '-c', 'airflow run deploy_site action_xcom \
|
|
2018-04-11T07:30:37 --job_id 2 --raw -sd DAGS_FOLDER/deploy_site.py']
|
|
Running on host airflow-worker-0.airflow-worker-discovery.ucp.svc.cluster.local
|
|
Subtask: [2018-04-11 07:30:43,944] {{python_operator.py:90}} \
|
|
INFO - Done. Returned value was: None
|
|
"""
|
|
|
|
|
|
class TestActionsStepsLogsEndpoint():
|
|
@patch.object(ActionsStepsLogsResource, 'get_action_step_logs',
|
|
common.str_responder)
|
|
def test_on_get(self, api_client):
|
|
"""Validate the on_get method returns 200 on success"""
|
|
# Define Endpoint
|
|
endpoint = "/api/v1.0/actions/{}/steps/{}/logs".format(
|
|
'01C9VVQSCFS7V9QB5GBS3WFVSE',
|
|
'action_xcom')
|
|
|
|
result = api_client.simulate_get(endpoint,
|
|
headers=common.AUTH_HEADERS)
|
|
assert result.status_code == 200
|
|
|
|
@patch('shipyard_airflow.control.helpers.action_helper.ActionsHelper',
|
|
autospec=True)
|
|
def test_generate_log_endpoint(self, mock_actions_helper):
|
|
"""Tests log endpoint generation"""
|
|
action_logs_resource = ActionsStepsLogsResource()
|
|
|
|
mock_actions_helper.get_formatted_dag_execution_date.return_value = (
|
|
EXECUTION_DATE_STR)
|
|
action_logs_resource.actions_helper = mock_actions_helper
|
|
|
|
# Define variables
|
|
action_id = ACTIONS_DB['name']
|
|
step = TASK_INSTANCE_DB[0]
|
|
step_id = TASK_INSTANCE_DB[0]['task_id']
|
|
try_number = '2'
|
|
required_endpoint = (
|
|
'http://airflow-worker-0.ucp.svc.cluster.local:8793/'
|
|
'log/deploy_site/action_xcom/2018-04-05T16:29:11/2.log')
|
|
|
|
log_endpoint = action_logs_resource.generate_log_endpoint(
|
|
step, action_id, step_id, try_number)
|
|
|
|
assert log_endpoint == required_endpoint
|
|
|
|
mock_actions_helper.get_formatted_dag_execution_date.\
|
|
assert_called_once_with(step)
|
|
|
|
def _mock_response(
|
|
self,
|
|
status=200,
|
|
text="TEXT"):
|
|
|
|
mock_resp = mock.Mock()
|
|
|
|
# set status code and content
|
|
mock_resp.status_code = status
|
|
mock_resp.text = text
|
|
|
|
return mock_resp
|
|
|
|
@mock.patch('requests.get')
|
|
def test_retrieve_logs(self, mock_get):
|
|
"""Tests log retrieval"""
|
|
action_logs_resource = ActionsStepsLogsResource()
|
|
|
|
# Define variables
|
|
log_endpoint = (
|
|
'http://airflow-worker-0.ucp.svc.cluster.local:8793/'
|
|
'log/deploy_site/action_xcom/2018-04-05T16:29:11/2.log')
|
|
|
|
mock_resp = self._mock_response(text=XCOM_RUN_LOGS)
|
|
mock_get.return_value = mock_resp
|
|
|
|
result = action_logs_resource.retrieve_logs(log_endpoint)
|
|
|
|
assert result == XCOM_RUN_LOGS
|
|
|
|
@mock.patch('requests.get')
|
|
def test_retrieve_logs_404(self, mock_get):
|
|
mock_get.return_value.status_code = 404
|
|
action_logs_resource = ActionsStepsLogsResource()
|
|
with pytest.raises(ApiError) as e:
|
|
action_logs_resource.retrieve_logs(None)
|
|
assert ('Airflow endpoint returned error status code' in
|
|
e.value.description)
|
|
assert falcon.HTTP_404 == e.value.status
|
|
|
|
@mock.patch('requests.get')
|
|
def test_retrieve_logs_error(self, mock_get):
|
|
mock_get.side_effect = requests.exceptions.ConnectionError
|
|
action_logs_resource = ActionsStepsLogsResource()
|
|
with pytest.raises(ApiError) as e:
|
|
action_logs_resource.retrieve_logs(None)
|
|
assert ("Exception happened during Airflow API request" in
|
|
e.value.description)
|
|
assert falcon.HTTP_500 == e.value.status
|