Add build data for MAAS logs

- Collect logs from MAAS when failures happen
  during deployment
- Save the logs to build data so it is available via
  API
- Add postgres integration test

Change-Id: Ied2d8539fe02a75f1f175a421b897b4f8ce07c8d

Makefile

@@ -51,7 +51,12 @@ coverage_test: build_drydock external_dep
 # Run just unit tests
 .PHONY: unit_tests
 unit_tests: external_dep
-	tox -re py35
+	tox -re py35 $(TESTS)
+
+# Run just DB integration tests
+.PHONY: db_integration_tests
+db_integration_tests: external_dep
+	tox -re integration $(TESTS)
 
 # Freeze full set of Python requirements
 .PHONY: req_freeze

drydock_provisioner/control/api.py

@@ -20,6 +20,7 @@ from .designs import DesignsPartsKindsResource
 from .designs import DesignsPartResource
 from .tasks import TasksResource
 from .tasks import TaskResource
+from .tasks import TaskBuilddataResource
 from .nodes import NodesResource
 from .nodes import NodeBuildDataResource
 from .nodes import NodeFilterResource
@@ -67,6 +68,8 @@ def start_api(state_manager=None, ingester=None, orchestrator=None):
          TasksResource(state_manager=state_manager,
                        orchestrator=orchestrator)),
         ('/tasks/{task_id}', TaskResource(state_manager=state_manager)),
+        ('/tasks/{task_id}/builddata',
+         TaskBuilddataResource(state_manager=state_manager)),
 
         # API for managing site design data
         ('/designs', DesignsResource(state_manager=state_manager)),

drydock_provisioner/control/tasks.py

@@ -390,3 +390,20 @@ class TaskResource(StatefulResource):
             # Finished this layer, incrementing for the next while loop.
             current_layer = current_layer + 1
         return resp_data, errors
+
+
+class TaskBuilddataResource(StatefulResource):
+    """Handler resource for /tasks/<id>/builddata singleton endpoint."""
+
+    @policy.ApiEnforcer('physical_provisioner:read_build_data')
+    def on_get(self, req, resp, task_id):
+        try:
+            bd_list = self.state_manager.get_build_data(task_id=task_id)
+            if not bd_list:
+                resp.status = falcon.HTTP_404
+                return
+            resp.body = json.dumps(bd_list)
+        except Exception as e:
+            resp.body = "Unexpected error."
+            resp.status = falcon.HTTP_500
+        resp.status = falcon.HTTP_200

drydock_provisioner/control/util.py

@@ -16,11 +16,13 @@ from drydock_provisioner.error import ApiError
 from drydock_provisioner.drydock_client.session import KeystoneClient
 from drydock_provisioner.util import KeystoneUtils
 
+
 def get_internal_api_href(ver):
     """Get the internal API href for Drydock API version ``ver``."""
 
     # TODO(sh8121att) Support versioned service registration
     supported_versions = ['v1.0']
+
     if ver in supported_versions:
         ks_sess = KeystoneUtils.get_session()
         url = KeystoneClient.get_endpoint(

drydock_provisioner/drivers/node/maasdriver/actions/node.py

@@ -27,6 +27,7 @@ import drydock_provisioner.objects.fields as hd_fields
 import drydock_provisioner.objects.hostprofile as hostprofile
 import drydock_provisioner.objects as objects
 
+from drydock_provisioner.control.util import get_internal_api_href
 from drydock_provisioner.orchestrator.actions.orchestrator import BaseAction
 
 import drydock_provisioner.drivers.node.maasdriver.models.fabric as maas_fabric
@@ -51,6 +52,22 @@ class BaseMaasAction(BaseAction):
         self.logger = logging.getLogger(
             config.config_mgr.conf.logging.nodedriver_logger_name)
 
+    def _add_detail_logs(self, node, machine, data_gen, result_type='all'):
+        result_details = machine.get_task_results(result_type=result_type)
+        for r in result_details:
+            bd = objects.BuildData(
+                node_name=node.name,
+                task_id=self.task.task_id,
+                collected_date=r.updated,
+                generator=data_gen,
+                data_format='text/plain',
+                data_element=r.get_decoded_data())
+            self.state_manager.post_build_data(bd)
+        log_href = "%s/tasks/%s/builddata" % (
+            get_internal_api_href("v1.0"), str(self.task.task_id))
+        self.task.result.add_link('detail_logs', log_href)
+        self.task.save()
+
 
 class ValidateNodeServices(BaseMaasAction):
     """Action to validate MaaS is available and ready for use."""
@@ -983,6 +1000,25 @@ class ConfigureHardware(BaseMaasAction):
                                 ctx_type='node')
                             self.task.success(focus=n.get_id())
                             self.collect_build_data(machine)
+                        else:
+                            msg = "Node %s failed commissioning." % (n.name)
+                            self.logger.info(msg)
+                            self.task.add_status_msg(
+                                msg=msg,
+                                error=True,
+                                ctx=n.name,
+                                ctx_type='node')
+                            self.task.failure(focus=n.get_id())
+                        self._add_detail_logs(
+                            n,
+                            machine,
+                            'maas_commission_log',
+                            result_type='commissioning')
+                        self._add_detail_logs(
+                            n,
+                            machine,
+                            'maas_testing_log',
+                            result_type='testing')
                     elif machine.status_name in ['Commissioning', 'Testing']:
                         msg = "Located node %s in MaaS, node already being commissioned. Skipping..." % (
                             n.name)
@@ -2123,6 +2159,12 @@ class DeployNode(BaseMaasAction):
                 self.task.add_status_msg(
                     msg=msg, error=False, ctx=n.name, ctx_type='node')
                 self.task.success(focus=n.get_id())
+            elif machine.status_name.startswith('Failed'):
+                msg = "Node %s deployment failed" % (n.name)
+                self.logger.info(msg)
+                self.task.add_status_msg(
+                    msg=msg, error=True, ctx=n.name, ctx_type='node')
+                self.task.failure(focus=n.get_id())
             else:
                 msg = "Node %s deployment timed out" % (n.name)
                 self.logger.warning(msg)
@@ -2130,6 +2172,9 @@ class DeployNode(BaseMaasAction):
                     msg=msg, error=True, ctx=n.name, ctx_type='node')
                 self.task.failure(focus=n.get_id())
 
+            self._add_detail_logs(
+                n, machine, 'maas_deploy_log', result_type='deploy')
+
         self.task.set_status(hd_fields.TaskStatus.Complete)
         self.task.save()
 

drydock_provisioner/drivers/node/maasdriver/models/machine.py

@@ -304,7 +304,9 @@ class Machine(model_base.ResourceBase):
                          ``all``, ``commissioning``, ``testing``, ``deploy``
         """
         node_results = maas_nr.NodeResults(
-            system_id_list=[self.resource_id], result_type=result_type)
+            self.api_client,
+            system_id_list=[self.resource_id],
+            result_type=result_type)
         node_results.refresh()
 
         return node_results

tests/integration/postgres/test_noderesult_links.py

@@ -0,0 +1,85 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Test build data collection and persistence."""
+from drydock_provisioner.objects import fields as hd_fields
+from drydock_provisioner.drivers.node.maasdriver.actions.node import BaseMaasAction
+from drydock_provisioner.drivers.node.maasdriver.models.machine import Machine
+
+
+class TestNodeResultLinks(object):
+    def test_create_detail_log_links(self, setup, blank_state, mocker,
+                                     input_files, deckhand_orchestrator):
+        """Test that the detail log collection from MaaS works."""
+
+        class MockedResponse():
+
+            status_code = 200
+
+            def json(self):
+                resp_content = [{
+                    "id":
+                    3,
+                    "data":
+                    "SGVsbG8gV29ybGQh",
+                    "result_type":
+                    0,
+                    "script_result":
+                    0,
+                    "resource_uri":
+                    "/MAAS/api/2.0/commissioning-scripts/",
+                    "updated":
+                    "2018-07-06T14:32:20.129",
+                    "node": {
+                        "system_id": "r7mqnw"
+                    },
+                    "created":
+                    "2018-07-06T14:37:12.632",
+                    "name":
+                    "hello_world"
+                }]
+
+                return resp_content
+
+        input_file = input_files.join("deckhand_fullsite.yaml")
+        design_ref = "file://%s" % str(input_file)
+
+        task = deckhand_orchestrator.create_task(
+            action=hd_fields.OrchestratorAction.Noop, design_ref=design_ref)
+        task.set_status(hd_fields.TaskStatus.Running)
+        task.save()
+
+        api_client = mocker.MagicMock()
+        api_client.get.return_value = MockedResponse()
+
+        machine = Machine(api_client)
+        machine.resource_id = 'r7mqnw'
+        node = mocker.MagicMock()
+        node.configure_mock(name='n1')
+
+        action = BaseMaasAction(task, deckhand_orchestrator, blank_state)
+
+        with mocker.patch(
+                'drydock_provisioner.drivers.node.maasdriver.actions.node.get_internal_api_href',
+                mocker.MagicMock(return_value='http://drydock/api/v1.0')):
+            action._add_detail_logs(node, machine, 'hello_world')
+
+        bd = blank_state.get_build_data(task_id=task.task_id)
+        assert len(bd) == 1
+
+        links_list = task.result.get_links()
+
+        assert len(links_list) > 0
+
+        for l in links_list:
+            assert str(task.task_id) in l

tests/unit/test_maasdriver_noderesults.py

@@ -18,6 +18,7 @@ from drydock_provisioner.drivers.node.maasdriver.models.node_results import Node
 class TestMaasNodeResults():
     def test_get_noderesults(self, mocker):
         '''Test noderesults refresh call to load a list of NodeResults.'''
+
         # A object to return that looks like a requests response
         # object wrapping a MAAS API response
         class MockedResponse():
@@ -26,17 +27,25 @@ class TestMaasNodeResults():
 
             def json(self):
                 resp_content = [{
-                    "id": 3,
+                    "id":
-                    "data": "SGVsbG8gV29ybGQh",
+                    3,
-                    "result_type": 0,
+                    "data":
-                    "script_result": 0,
+                    "SGVsbG8gV29ybGQh",
-                    "resource_uri": "/MAAS/api/2.0/commissioning-scripts/",
+                    "result_type":
-                    "updated": "2018-07-06T14:32:20.129",
+                    0,
+                    "script_result":
+                    0,
+                    "resource_uri":
+                    "/MAAS/api/2.0/commissioning-scripts/",
+                    "updated":
+                    "2018-07-06T14:32:20.129",
                     "node": {
                         "system_id": "r7mqnw"
                     },
-                    "created": "2018-07-06T14:37:12.632",
+                    "created":
-                    "name": "hello_world"
+                    "2018-07-06T14:37:12.632",
+                    "name":
+                    "hello_world"
                 }]
 
                 return resp_content