From f57301fae9171202af8a75458145cf0bc74764d4 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Mon, 9 Jul 2018 14:54:52 -0500 Subject: [PATCH] Add build data for MAAS logs - Collect logs from MAAS when failures happen during deployment - Save the logs to build data so it is available via API - Add postgres integration test Change-Id: Ied2d8539fe02a75f1f175a421b897b4f8ce07c8d --- Makefile | 7 +- drydock_provisioner/control/api.py | 3 + drydock_provisioner/control/tasks.py | 17 ++++ drydock_provisioner/control/util.py | 2 + .../drivers/node/maasdriver/actions/node.py | 45 ++++++++++ .../drivers/node/maasdriver/models/machine.py | 4 +- .../postgres/test_noderesult_links.py | 85 +++++++++++++++++++ tests/unit/test_maasdriver_noderesults.py | 25 ++++-- 8 files changed, 178 insertions(+), 10 deletions(-) create mode 100644 tests/integration/postgres/test_noderesult_links.py diff --git a/Makefile b/Makefile index a1784da7..fd974d19 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,12 @@ coverage_test: build_drydock external_dep # Run just unit tests .PHONY: unit_tests unit_tests: external_dep - tox -re py35 + tox -re py35 $(TESTS) + +# Run just DB integration tests +.PHONY: db_integration_tests +db_integration_tests: external_dep + tox -re integration $(TESTS) # Freeze full set of Python requirements .PHONY: req_freeze diff --git a/drydock_provisioner/control/api.py b/drydock_provisioner/control/api.py index 2c1b0805..beafd5b7 100644 --- a/drydock_provisioner/control/api.py +++ b/drydock_provisioner/control/api.py @@ -20,6 +20,7 @@ from .designs import DesignsPartsKindsResource from .designs import DesignsPartResource from .tasks import TasksResource from .tasks import TaskResource +from .tasks import TaskBuilddataResource from .nodes import NodesResource from .nodes import NodeBuildDataResource from .nodes import NodeFilterResource @@ -67,6 +68,8 @@ def start_api(state_manager=None, ingester=None, orchestrator=None): TasksResource(state_manager=state_manager, orchestrator=orchestrator)), ('/tasks/{task_id}', TaskResource(state_manager=state_manager)), + ('/tasks/{task_id}/builddata', + TaskBuilddataResource(state_manager=state_manager)), # API for managing site design data ('/designs', DesignsResource(state_manager=state_manager)), diff --git a/drydock_provisioner/control/tasks.py b/drydock_provisioner/control/tasks.py index e83094fe..fc4d4f2f 100644 --- a/drydock_provisioner/control/tasks.py +++ b/drydock_provisioner/control/tasks.py @@ -390,3 +390,20 @@ class TaskResource(StatefulResource): # Finished this layer, incrementing for the next while loop. current_layer = current_layer + 1 return resp_data, errors + + +class TaskBuilddataResource(StatefulResource): + """Handler resource for /tasks//builddata singleton endpoint.""" + + @policy.ApiEnforcer('physical_provisioner:read_build_data') + def on_get(self, req, resp, task_id): + try: + bd_list = self.state_manager.get_build_data(task_id=task_id) + if not bd_list: + resp.status = falcon.HTTP_404 + return + resp.body = json.dumps(bd_list) + except Exception as e: + resp.body = "Unexpected error." + resp.status = falcon.HTTP_500 + resp.status = falcon.HTTP_200 diff --git a/drydock_provisioner/control/util.py b/drydock_provisioner/control/util.py index d5b9d840..371da1c8 100644 --- a/drydock_provisioner/control/util.py +++ b/drydock_provisioner/control/util.py @@ -16,11 +16,13 @@ from drydock_provisioner.error import ApiError from drydock_provisioner.drydock_client.session import KeystoneClient from drydock_provisioner.util import KeystoneUtils + def get_internal_api_href(ver): """Get the internal API href for Drydock API version ``ver``.""" # TODO(sh8121att) Support versioned service registration supported_versions = ['v1.0'] + if ver in supported_versions: ks_sess = KeystoneUtils.get_session() url = KeystoneClient.get_endpoint( diff --git a/drydock_provisioner/drivers/node/maasdriver/actions/node.py b/drydock_provisioner/drivers/node/maasdriver/actions/node.py index 995a5211..9801c7b3 100644 --- a/drydock_provisioner/drivers/node/maasdriver/actions/node.py +++ b/drydock_provisioner/drivers/node/maasdriver/actions/node.py @@ -27,6 +27,7 @@ import drydock_provisioner.objects.fields as hd_fields import drydock_provisioner.objects.hostprofile as hostprofile import drydock_provisioner.objects as objects +from drydock_provisioner.control.util import get_internal_api_href from drydock_provisioner.orchestrator.actions.orchestrator import BaseAction import drydock_provisioner.drivers.node.maasdriver.models.fabric as maas_fabric @@ -51,6 +52,22 @@ class BaseMaasAction(BaseAction): self.logger = logging.getLogger( config.config_mgr.conf.logging.nodedriver_logger_name) + def _add_detail_logs(self, node, machine, data_gen, result_type='all'): + result_details = machine.get_task_results(result_type=result_type) + for r in result_details: + bd = objects.BuildData( + node_name=node.name, + task_id=self.task.task_id, + collected_date=r.updated, + generator=data_gen, + data_format='text/plain', + data_element=r.get_decoded_data()) + self.state_manager.post_build_data(bd) + log_href = "%s/tasks/%s/builddata" % ( + get_internal_api_href("v1.0"), str(self.task.task_id)) + self.task.result.add_link('detail_logs', log_href) + self.task.save() + class ValidateNodeServices(BaseMaasAction): """Action to validate MaaS is available and ready for use.""" @@ -983,6 +1000,25 @@ class ConfigureHardware(BaseMaasAction): ctx_type='node') self.task.success(focus=n.get_id()) self.collect_build_data(machine) + else: + msg = "Node %s failed commissioning." % (n.name) + self.logger.info(msg) + self.task.add_status_msg( + msg=msg, + error=True, + ctx=n.name, + ctx_type='node') + self.task.failure(focus=n.get_id()) + self._add_detail_logs( + n, + machine, + 'maas_commission_log', + result_type='commissioning') + self._add_detail_logs( + n, + machine, + 'maas_testing_log', + result_type='testing') elif machine.status_name in ['Commissioning', 'Testing']: msg = "Located node %s in MaaS, node already being commissioned. Skipping..." % ( n.name) @@ -2123,6 +2159,12 @@ class DeployNode(BaseMaasAction): self.task.add_status_msg( msg=msg, error=False, ctx=n.name, ctx_type='node') self.task.success(focus=n.get_id()) + elif machine.status_name.startswith('Failed'): + msg = "Node %s deployment failed" % (n.name) + self.logger.info(msg) + self.task.add_status_msg( + msg=msg, error=True, ctx=n.name, ctx_type='node') + self.task.failure(focus=n.get_id()) else: msg = "Node %s deployment timed out" % (n.name) self.logger.warning(msg) @@ -2130,6 +2172,9 @@ class DeployNode(BaseMaasAction): msg=msg, error=True, ctx=n.name, ctx_type='node') self.task.failure(focus=n.get_id()) + self._add_detail_logs( + n, machine, 'maas_deploy_log', result_type='deploy') + self.task.set_status(hd_fields.TaskStatus.Complete) self.task.save() diff --git a/drydock_provisioner/drivers/node/maasdriver/models/machine.py b/drydock_provisioner/drivers/node/maasdriver/models/machine.py index 69a0294c..d92fa661 100644 --- a/drydock_provisioner/drivers/node/maasdriver/models/machine.py +++ b/drydock_provisioner/drivers/node/maasdriver/models/machine.py @@ -304,7 +304,9 @@ class Machine(model_base.ResourceBase): ``all``, ``commissioning``, ``testing``, ``deploy`` """ node_results = maas_nr.NodeResults( - system_id_list=[self.resource_id], result_type=result_type) + self.api_client, + system_id_list=[self.resource_id], + result_type=result_type) node_results.refresh() return node_results diff --git a/tests/integration/postgres/test_noderesult_links.py b/tests/integration/postgres/test_noderesult_links.py new file mode 100644 index 00000000..e51c840d --- /dev/null +++ b/tests/integration/postgres/test_noderesult_links.py @@ -0,0 +1,85 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test build data collection and persistence.""" +from drydock_provisioner.objects import fields as hd_fields +from drydock_provisioner.drivers.node.maasdriver.actions.node import BaseMaasAction +from drydock_provisioner.drivers.node.maasdriver.models.machine import Machine + + +class TestNodeResultLinks(object): + def test_create_detail_log_links(self, setup, blank_state, mocker, + input_files, deckhand_orchestrator): + """Test that the detail log collection from MaaS works.""" + + class MockedResponse(): + + status_code = 200 + + def json(self): + resp_content = [{ + "id": + 3, + "data": + "SGVsbG8gV29ybGQh", + "result_type": + 0, + "script_result": + 0, + "resource_uri": + "/MAAS/api/2.0/commissioning-scripts/", + "updated": + "2018-07-06T14:32:20.129", + "node": { + "system_id": "r7mqnw" + }, + "created": + "2018-07-06T14:37:12.632", + "name": + "hello_world" + }] + + return resp_content + + input_file = input_files.join("deckhand_fullsite.yaml") + design_ref = "file://%s" % str(input_file) + + task = deckhand_orchestrator.create_task( + action=hd_fields.OrchestratorAction.Noop, design_ref=design_ref) + task.set_status(hd_fields.TaskStatus.Running) + task.save() + + api_client = mocker.MagicMock() + api_client.get.return_value = MockedResponse() + + machine = Machine(api_client) + machine.resource_id = 'r7mqnw' + node = mocker.MagicMock() + node.configure_mock(name='n1') + + action = BaseMaasAction(task, deckhand_orchestrator, blank_state) + + with mocker.patch( + 'drydock_provisioner.drivers.node.maasdriver.actions.node.get_internal_api_href', + mocker.MagicMock(return_value='http://drydock/api/v1.0')): + action._add_detail_logs(node, machine, 'hello_world') + + bd = blank_state.get_build_data(task_id=task.task_id) + assert len(bd) == 1 + + links_list = task.result.get_links() + + assert len(links_list) > 0 + + for l in links_list: + assert str(task.task_id) in l diff --git a/tests/unit/test_maasdriver_noderesults.py b/tests/unit/test_maasdriver_noderesults.py index 954a7e2a..4e6963b7 100644 --- a/tests/unit/test_maasdriver_noderesults.py +++ b/tests/unit/test_maasdriver_noderesults.py @@ -18,6 +18,7 @@ from drydock_provisioner.drivers.node.maasdriver.models.node_results import Node class TestMaasNodeResults(): def test_get_noderesults(self, mocker): '''Test noderesults refresh call to load a list of NodeResults.''' + # A object to return that looks like a requests response # object wrapping a MAAS API response class MockedResponse(): @@ -26,17 +27,25 @@ class TestMaasNodeResults(): def json(self): resp_content = [{ - "id": 3, - "data": "SGVsbG8gV29ybGQh", - "result_type": 0, - "script_result": 0, - "resource_uri": "/MAAS/api/2.0/commissioning-scripts/", - "updated": "2018-07-06T14:32:20.129", + "id": + 3, + "data": + "SGVsbG8gV29ybGQh", + "result_type": + 0, + "script_result": + 0, + "resource_uri": + "/MAAS/api/2.0/commissioning-scripts/", + "updated": + "2018-07-06T14:32:20.129", "node": { "system_id": "r7mqnw" }, - "created": "2018-07-06T14:37:12.632", - "name": "hello_world" + "created": + "2018-07-06T14:37:12.632", + "name": + "hello_world" }] return resp_content