airship/drydock
Code Issues Proposed changes

Add build data support to the API

Browse Source 
- Add list of build data to detailed task response when builddata=true
  specifed in query string
- Add new endpoint for /nodes/nodename/builddata to retrieve
  build data for a particular node
- Update docs for new API capabilities
- Testing all around

Change-Id: If0fcd2962d4389789af45ad1fbe61d226ac6a403
This commit is contained in:
Scott Hussey 2017-12-18 16:47:58 -06:00
parent a20ecbfba0
commit 74ce4aaef0
9 changed files with 331 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
docs/source/API.rst
View File

@ -20,6 +20,34 @@ tasks API
The Tasks API is used for creating and listing asynchronous tasks to be executed by the
Drydock orchestrator. See :ref:`task` for details on creating tasks and field information.
nodes API
---------
GET nodes
^^^^^^^^^
The Nodes API will provide a report of current nodes as known by the node provisioner
and their status with a few hardware details.
GET nodes/hostname/builddata
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Get all the build data record for node ``hostname``. The response will be a list of
objects in the below form.::
{
"node_name": "hostname",
"generator": "description of how data was generated",
"collected_date": ios8601 UTC datestamp,
"task_id": "UUID of task initiating collection",
"data_format": "MIME-type of data_element",
"data_element": "Collected data"
}
If the query parameter ``latest`` is passed with a value of ``true``, then only
the most recently collected data for each ``generator`` will be included in the
response.
bootdata
--------

22
docs/source/task.rst
View File

@ -88,7 +88,7 @@ When querying the state of an existing task, the below document will be returned
{
"Kind": "Task",
"apiVersion": "v1",
"apiVersion": "v1.0",
"task_id": "uuid",
"action": "validate_design|verify_site|prepare_site|verify_node|prepare_node|deploy_node|destroy_node",
"design_ref": "http_uri|deckhand_uri|file_uri",
@ -144,4 +144,24 @@ consist of the below::
"ts": iso8601 UTC timestamp,
}
Task Build Data
~~~~~~~~~~~~~~~
When querying the detail state of an existing task, adding the parameter ``builddata=true``
in the query string will add one additional field with a list of build data elements
collected by this task.::
{
"Kind": "Task",
"apiVersion": "v1",
....
"build_data": [
{
"node_name": "foo",
"task_id": "uuid",
"collected_data": iso8601 UTC timestamp,
"generator": "lshw",
"data_format": "application/json",
"data_element": "{ \"id\": \"foo\", \"class\": \"system\" ...}"
}
]

4
drydock_provisioner/control/api.py
View File

@ -21,6 +21,7 @@ from .designs import DesignsPartResource
from .tasks import TasksResource
from .tasks import TaskResource
from .nodes import NodesResource
from .nodes import NodeBuildDataResource
from .health import HealthResource
from .bootaction import BootactionUnitsResource
from .bootaction import BootactionFilesResource
@ -75,6 +76,9 @@ def start_api(state_manager=None, ingester=None, orchestrator=None):
# API to list current MaaS nodes
('/nodes', NodesResource()),
# API to get build data for a node
('/nodes/{hostname}/builddata',
NodeBuildDataResource(state_manager=state_manager)),
# API for nodes to discover their boot actions during curtin install
('/bootactions/nodes/{hostname}/units',
BootactionUnitsResource(

31
drydock_provisioner/control/nodes.py
View File

@ -20,7 +20,7 @@ from drydock_provisioner import config
from drydock_provisioner.drivers.node.maasdriver.api_client import MaasRequestFactory
from drydock_provisioner.drivers.node.maasdriver.models.machine import Machines
from .base import BaseResource
from .base import BaseResource, StatefulResource
class NodesResource(BaseResource):
@ -57,3 +57,32 @@ class NodesResource(BaseResource):
self.error(req.context, "Unknown error: %s" % str(ex), exc_info=ex)
self.return_error(
resp, falcon.HTTP_500, message="Unknown error", retry=False)
class NodeBuildDataResource(StatefulResource):
"""Resource for returning build data for a node."""
@policy.ApiEnforcer('physical_provisioner:read_build_data')
def on_get(self, req, resp, hostname):
try:
latest = req.params.get('latest', 'false').upper()
latest = True if latest == 'TRUE' else False
node_bd = self.state_manager.get_build_data(
node_name=hostname, latest=latest)
if not node_bd:
self.return_error(
resp,
falcon.HTTP_404,
message="No build data found",
retry=False)
else:
node_bd = [bd.to_dict() for bd in node_bd]
resp.status = falcon.HTTP_200
resp.body = json.dumps(node_bd)
resp.content_type = falcon.MEDIA_JSON
except Exception as ex:
self.error(req.context, "Unknown error: %s" % str(ex), exc_info=ex)
self.return_error(
resp, falcon.HTTP_500, message="Unknown error", retry=False)

11
drydock_provisioner/control/tasks.py
View File

@ -300,7 +300,6 @@ class TaskResource(StatefulResource):
"""Handler for GET method."""
try:
task = self.state_manager.get_task(uuid.UUID(task_id))
if task is None:
self.info(req.context, "Task %s does not exist" % task_id)
self.return_error(
@ -310,7 +309,15 @@ class TaskResource(StatefulResource):
retry=False)
return
resp.body = json.dumps(task.to_dict())
resp_data = task.to_dict()
builddata = req.params.get('builddata', 'false').upper()
if builddata == "TRUE":
task_bd = self.state_manager.get_build_data(
task_id=task.get_id())
resp_data['build_data'] = [bd.to_dict() for bd in task_bd]
resp.body = json.dumps(resp_data)
resp.status = falcon.HTTP_200
except Exception as ex:
self.error(req.context, "Unknown error: %s" % (str(ex)))

2
drydock_provisioner/objects/task.py
View File

@ -558,7 +558,7 @@ class TaskStatus(object):
def to_dict(self):
return {
'kind': 'Status',
'apiVersion': 'v1',
'apiVersion': 'v1.0',
'metadata': {},
'message': self.message,
'reason': self.reason,

7
drydock_provisioner/policy.py
View File

@ -95,6 +95,13 @@ class DrydockPolicy(object):
'path': '/api/v1.0/tasks',
'method': 'POST'
}]),
policy.DocumentedRuleDefault(
'physical_provisioner:read_build_data', 'role:admin',
'Read build data for a node',
[{
'path': '/api/v1.0/nodes/{nodename}/builddata',
'method': 'GET',
}]),
]
# Data Management Policy

152
tests/integration/postgres/test_api_builddata.py Normal file
View File

@ -0,0 +1,152 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test nodes build data API with Postgres backend."""
import pytest
from falcon import testing
import uuid
import datetime
import random
import drydock_provisioner.objects as objects
from drydock_provisioner import policy
from drydock_provisioner.control.api import start_api
import falcon
class TestNodeBuildDataApi():
def test_read_builddata_all(self, falcontest, seeded_builddata):
"""Test that by default the API returns all build data for a node."""
url = '/api/v1.0/nodes/foo/builddata'
# Seed the database with build data
nodelist = ['foo']
count = 3
seeded_builddata(nodelist=nodelist, count=count)
# TODO(sh8121att) Make fixture for request header forging
hdr = {
'Content-Type': 'application/json',
'X-IDENTITY-STATUS': 'Confirmed',
'X-USER-NAME': 'Test',
'X-ROLES': 'admin'
}
resp = falcontest.simulate_get(url, headers=hdr)
assert resp.status == falcon.HTTP_200
resp_body = resp.json
assert len(resp_body) == count
def test_read_builddata_latest(self, falcontest, seeded_builddata):
"""Test that the ``latest`` parameter works."""
url = '/api/v1.0/nodes/foo/builddata'
req_hdr = {
'Content-Type': 'application/json',
'X-IDENTITY-STATUS': 'Confirmed',
'X-USER-NAME': 'Test',
'X-ROLES': 'admin',
}
# Seed the database with build data
nodelist = ['foo']
generatorlist = ['hello', 'hello', 'bye']
count = 3
seeded_builddata(
nodelist=nodelist,
generatorlist=generatorlist,
count=count,
random_dates=True)
resp = falcontest.simulate_get(
url, headers=req_hdr, query_string="latest=true")
assert resp.status == falcon.HTTP_200
resp_body = resp.json
# Should only be a single instance for each unique generator
assert len(resp_body) == len(set(generatorlist))
@pytest.fixture()
def seeded_builddata(self, blank_state):
"""Provide function to seed the database with build data."""
def seed_build_data(nodelist=['foo'],
tasklist=None,
generatorlist=None,
count=1,
random_dates=True):
"""Seed the database with ``count`` build data elements for each node.
If tasklist is specified, it should be a list of length ``count`` such that
as build_data are added for a node, each task_id will be used one for each node
:param nodelist: list of string nodenames for build data
:param tasklist: list of uuid.UUID IDs for task. If omitted, uuids will be generated
:param gneratorlist: list of string generatos to assign to build data. If omitted, 'hello_world'
is used.
:param count: number build data elements to create for each node
:param random_dates: whether to generate random dates in the past 180 days or create each
build data element with utcnow()
"""
for n in nodelist:
for i in range(count):
if random_dates:
collected_date = datetime.datetime.utcnow(
) - datetime.timedelta(days=random.randint(1, 180))
else:
collected_date = None
if tasklist:
task_id = tasklist[i]
else:
task_id = uuid.uuid4()
if generatorlist:
generator = generatorlist[i]
else:
generator = 'hello_world'
bd = objects.BuildData(
node_name=n,
task_id=task_id,
generator=generator,
data_format='text/plain',
collected_date=collected_date,
data_element='Hello World!')
blank_state.post_build_data(bd)
i = i + 1
return seed_build_data
# TODO(sh8121att) Make this a general fixture in conftest.py
@pytest.fixture()
def falcontest(self, drydock_state, deckhand_ingester,
deckhand_orchestrator):
"""Create a test harness for the the Falcon API framework."""
policy.policy_engine = policy.DrydockPolicy()
policy.policy_engine.register_policy()
return testing.TestClient(
start_api(
state_manager=drydock_state,
ingester=deckhand_ingester,
orchestrator=deckhand_orchestrator))
@policy.ApiEnforcer('physical_provisioner:read_task')
def target_function(self, req, resp):
return True

148
tests/integration/postgres/test_api_tasks.py
View File

@ -12,96 +12,106 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test tasks API with Postgres backend."""
import pytest
from falcon import testing
import uuid
import json
from drydock_provisioner import policy
from drydock_provisioner.orchestrator.orchestrator import Orchestrator
import drydock_provisioner.objects.fields as hd_fields
import drydock_provisioner.objects as objects
from drydock_provisioner import policy
from drydock_provisioner.control.api import start_api
from drydock_provisioner.control.base import DrydockRequestContext
from drydock_provisioner.control.tasks import TasksResource
import falcon
class TestTasksApi():
def test_read_tasks(self, mocker, blank_state):
''' DrydockPolicy.authorized() should correctly use oslo_policy to enforce
RBAC policy based on a DrydockRequestContext instance
'''
def test_read_tasks(self, falcontest, blank_state):
"""Test that the tasks API responds with list of tasks."""
url = '/api/v1.0/tasks'
mocker.patch('oslo_policy.policy.Enforcer')
# TODO(sh8121att) Make fixture for request header forging
hdr = {
'Content-Type': 'application/json',
'X-IDENTITY-STATUS': 'Confirmed',
'X-USER-NAME': 'Test',
'X-ROLES': 'admin'
}
ctx = DrydockRequestContext()
policy_engine = policy.DrydockPolicy()
# Mock policy enforcement
policy_mock_config = {'authorize.return_value': True}
policy_engine.enforcer.configre_mock(**policy_mock_config)
api = TasksResource(state_manager=blank_state)
# Configure context
project_id = str(uuid.uuid4().hex)
ctx.project_id = project_id
user_id = str(uuid.uuid4().hex)
ctx.user_id = user_id
ctx.roles = ['admin']
ctx.set_policy_engine(policy_engine)
# Configure mocked request and response
req = mocker.MagicMock()
resp = mocker.MagicMock()
req.context = ctx
api.on_get(req, resp)
resp = falcontest.simulate_get(url, headers=hdr)
assert resp.status == falcon.HTTP_200
def test_create_task(self, mocker, blank_state):
mocker.patch('oslo_policy.policy.Enforcer')
ingester = mocker.MagicMock()
orch = mocker.MagicMock(
spec=Orchestrator,
wraps=Orchestrator(state_manager=blank_state, ingester=ingester))
def test_read_tasks_builddata(self, falcontest, blank_state,
deckhand_orchestrator):
"""Test that the tasks API includes build data when prompted."""
req_hdr = {
'Content-Type': 'application/json',
'X-IDENTITY-STATUS': 'Confirmed',
'X-USER-NAME': 'Test',
'X-ROLES': 'admin',
}
# Seed DB with a task
ctx = DrydockRequestContext()
policy_engine = policy.DrydockPolicy()
task = deckhand_orchestrator.create_task(
action=hd_fields.OrchestratorAction.PrepareNodes,
design_ref='http://foo.com',
context=ctx)
# Seed DB with build data for task
build_data = objects.BuildData(
node_name='foo',
task_id=task.get_id(),
generator='hello_world',
data_format='text/plain',
data_element='Hello World!')
blank_state.post_build_data(build_data)
url = '/api/v1.0/tasks/%s' % str(task.get_id())
resp = falcontest.simulate_get(
url, headers=req_hdr, query_string="builddata=true")
assert resp.status == falcon.HTTP_200
resp_body = resp.json
assert isinstance(resp_body.get('build_data'), list)
assert len(resp_body.get('build_data')) == 1
def test_create_task(self, falcontest, blank_state):
url = '/api/v1.0/tasks'
req_hdr = {
'Content-Type': 'application/json',
'X-IDENTITY-STATUS': 'Confirmed',
'X-USER-NAME': 'Test',
'X-ROLES': 'admin',
}
json_body = json.dumps({
'action': 'verify_site',
'design_ref': 'http://foo.com',
}).encode('utf-8')
})
# Mock policy enforcement
policy_mock_config = {'authorize.return_value': True}
policy_engine.enforcer.configure_mock(**policy_mock_config)
api = TasksResource(orchestrator=orch, state_manager=blank_state)
# Configure context
project_id = str(uuid.uuid4().hex)
ctx.project_id = project_id
user_id = str(uuid.uuid4().hex)
ctx.user_id = user_id
ctx.roles = ['admin']
ctx.set_policy_engine(policy_engine)
# Configure mocked request and response
req = mocker.MagicMock(spec=falcon.Request)
req.content_type = 'application/json'
req.stream.read.return_value = json_body
resp = mocker.MagicMock(spec=falcon.Response)
req.context = ctx
api.on_post(req, resp)
resp = falcontest.simulate_post(url, headers=req_hdr, body=json_body)
assert resp.status == falcon.HTTP_201
assert resp.get_header('Location') is not None
assert resp.headers.get('Location') is not None
@policy.ApiEnforcer('physical_provisioner:read_task')
def target_function(self, req, resp):
return True
# TODO(sh8121att) Make this a general fixture in conftest.py
@pytest.fixture()
def falcontest(self, drydock_state, deckhand_ingester,
deckhand_orchestrator):
"""Create a test harness for the the Falcon API framework."""
policy.policy_engine = policy.DrydockPolicy()
policy.policy_engine.register_policy()
return testing.TestClient(
start_api(
state_manager=drydock_state,
ingester=deckhand_ingester,
orchestrator=deckhand_orchestrator))