Add notes common code for Shipyard
Adds the common/shared code to support notes in Shipyard such that this component can be reused between the Shipyard API and the workflows. Includes a single example of creating and retreiving notes as part of action creation, that will likely be changed in a future change. Note that this change doesn't include changes to the CLI to represent notes outwardly, but the first example can be seen using the --output-format=format option (shipyard get actions) Change-Id: I2f87713eb74dae312912ff4c36e6ae30a569ea38
This commit is contained in:
parent
44f7022df6
commit
35d6089d32
@ -369,6 +369,13 @@
|
||||
# Airship component validation timeout (in seconds) (integer value)
|
||||
#validation_read_timeout = 300
|
||||
|
||||
# Maximum time to wait to connect to a note source URL (in seconds) (integer
|
||||
# value)
|
||||
#notes_connect_timeout = 5
|
||||
|
||||
# Read timeout for a note source URL (in seconds) (integer value)
|
||||
#notes_read_timeout = 10
|
||||
|
||||
|
||||
[shipyard]
|
||||
|
||||
|
@ -0,0 +1,47 @@
|
||||
"""create notes table
|
||||
|
||||
Revision ID: 7486ddec1979
|
||||
Revises: 51b92375e5c4
|
||||
Create Date: 2018-10-02 16:10:08.139378
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import (types, func)
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '7486ddec1979'
|
||||
down_revision = '51b92375e5c4'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
'notes',
|
||||
# ULID key for the note
|
||||
sa.Column('note_id', types.String(26), primary_key=True),
|
||||
# The supplied association id used for lookup
|
||||
sa.Column('assoc_id', types.String(128), nullable=False),
|
||||
# The supplied subject of the note (what is this note about?)
|
||||
sa.Column('subject', types.String(128), nullable=False),
|
||||
# The supplied type of the subject (what kind of thing is the subject?)
|
||||
sa.Column('sub_type', types.String(128), nullable=False),
|
||||
# The text value of the note
|
||||
sa.Column('note_val', sa.Text, nullable=False),
|
||||
# The numeric verbosity level of the note (1-5)
|
||||
sa.Column('verbosity', types.Integer, nullable=False),
|
||||
# An optional URL containing more info for the note
|
||||
sa.Column('link_url', sa.Text, nullable=True),
|
||||
# Boolean if the link requires a X-Auth-Token header
|
||||
sa.Column('is_auth_link', types.Boolean, nullable=False),
|
||||
# The creation timestamp for the note
|
||||
sa.Column('note_timestamp',
|
||||
types.TIMESTAMP(timezone=True),
|
||||
server_default=func.now()),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table('notes')
|
@ -369,6 +369,13 @@
|
||||
# Airship component validation timeout (in seconds) (integer value)
|
||||
#validation_read_timeout = 300
|
||||
|
||||
# Maximum time to wait to connect to a note source URL (in seconds) (integer
|
||||
# value)
|
||||
#notes_connect_timeout = 5
|
||||
|
||||
# Read timeout for a note source URL (in seconds) (integer value)
|
||||
#notes_read_timeout = 10
|
||||
|
||||
|
||||
[shipyard]
|
||||
|
||||
|
@ -0,0 +1,44 @@
|
||||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
"""Errors for the Notes component"""
|
||||
|
||||
|
||||
class NotesError(Exception):
|
||||
"""Base exception for all NotesErrors"""
|
||||
pass
|
||||
|
||||
|
||||
class NotesInitializationError(NotesError):
|
||||
"""NotesInitializationError
|
||||
|
||||
Raised for errors while initializing a Notes instance
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class NotesRetrievalError(NotesError):
|
||||
"""NotesRetrievalError
|
||||
|
||||
Raised when there is an error retrieving notes
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class NotesStorageError(NotesError):
|
||||
"""NotesStorageError
|
||||
|
||||
Raised when there is an error attempting to store a note.
|
||||
"""
|
||||
pass
|
318
src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes.py
Normal file
318
src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes.py
Normal file
@ -0,0 +1,318 @@
|
||||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
"""Notes
|
||||
|
||||
A reusable component allowing for recording and retreiving information related
|
||||
to arbitrarily useage-based keys (the usage makes the association keys to their
|
||||
string-based liking). The intention for Notes is generally for additional
|
||||
non-fielded information that may be of interest to a user. Notes are not
|
||||
intended to store info that would drive code paths or decisions. This is not
|
||||
an arbitrary use key-value database.
|
||||
"""
|
||||
import abc
|
||||
from datetime import datetime
|
||||
import logging
|
||||
|
||||
import requests
|
||||
from requests.exceptions import HTTPError
|
||||
from requests.exceptions import RequestException
|
||||
import ulid
|
||||
|
||||
from .errors import NotesInitializationError
|
||||
from .errors import NotesRetrievalError
|
||||
from .errors import NotesStorageError
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
MAX_VERBOSITY = 5
|
||||
MIN_VERBOSITY = 1
|
||||
|
||||
|
||||
class NotesManager:
|
||||
"""Interface to store and retrieve notes
|
||||
|
||||
:param storage: A NotesStorage object to store and retrieve notes for a
|
||||
specific storage mechanism. e.g. Database, Service
|
||||
:param get_token: A method that returns an auth token that will be used
|
||||
as the X-Auth-Token header when resolving url-based notes
|
||||
:param connect_timeout: optional, The maximum time waiting to connect to
|
||||
a URL. Defaults to 3 seconds
|
||||
:param read_timeout: optional, The maximum time waiting to read the info
|
||||
from a URL. Defaults to 10 seconds
|
||||
|
||||
Example usage:
|
||||
nm = NotesManager(SQLNotesStorage("connection_info"), get_url)
|
||||
a_note = nm.store(Note(...params...))
|
||||
notes = list(nm.retrieve(Query("some/id")))
|
||||
"""
|
||||
def __init__(self, storage, get_token, connect_timeout=None,
|
||||
read_timeout=None):
|
||||
if not isinstance(storage, NotesStorage):
|
||||
raise NotesInitializationError(
|
||||
"Storage object is not suitable for use with Notes"
|
||||
)
|
||||
self.storage = storage
|
||||
LOG.info(
|
||||
"Initializing Notes with storage mechanism: %s",
|
||||
storage.__class__.__name__
|
||||
)
|
||||
|
||||
if not callable(get_token):
|
||||
raise NotesInitializationError(
|
||||
"Parameter get_token is not suitable for use with Notes. "
|
||||
"Must be a callable."
|
||||
)
|
||||
self.get_token = get_token
|
||||
|
||||
# connect and read timeouts default to 3 and 10 seconds
|
||||
self.connect_timeout = connect_timeout or 3
|
||||
self.read_timeout = read_timeout or 10
|
||||
|
||||
def create(self, assoc_id, subject, sub_type, note_val,
|
||||
verbosity=None, link_url=None, is_auth_link=None,
|
||||
note_id=None, note_timestamp=None, store=True):
|
||||
"""Creates and stores a Note object from parameters
|
||||
|
||||
Passthrough helper method to avoid additional imports for the Note
|
||||
class. Most of the parameters match that of the Note constructor.
|
||||
See: func:`notes.Note.__init__`
|
||||
Additional Parameters:
|
||||
|
||||
:param store: optinal, default=True, invoke the store method
|
||||
immediately upon creation, if true
|
||||
"""
|
||||
n = Note(assoc_id, subject, sub_type, note_val,
|
||||
verbosity=None, link_url=None, is_auth_link=None,
|
||||
note_id=None, note_timestamp=None)
|
||||
if store:
|
||||
return self.store(n)
|
||||
else:
|
||||
return n
|
||||
|
||||
def store(self, note):
|
||||
"""Store a note
|
||||
|
||||
:param note: A Note object to store
|
||||
:returns: The note, as it was after storage
|
||||
"""
|
||||
if note.verbosity < MIN_VERBOSITY or note.verbosity > MAX_VERBOSITY:
|
||||
raise NotesStorageError(
|
||||
"Verbosity of notes must range from {} "
|
||||
"to {} (most verbose)".format(MIN_VERBOSITY, MAX_VERBOSITY))
|
||||
try:
|
||||
return self.storage.store(note)
|
||||
except NotesStorageError:
|
||||
raise
|
||||
except Exception as ex:
|
||||
LOG.exception(ex)
|
||||
raise NotesStorageError("Unhandled error during storage of a note")
|
||||
|
||||
def retrieve(self, query):
|
||||
"""Retrieve a list of notes
|
||||
|
||||
:param query: a query object to retrieve notes
|
||||
"""
|
||||
try:
|
||||
notes = list(self.storage.retrieve(query))
|
||||
except NotesRetrievalError:
|
||||
raise
|
||||
except Exception as ex:
|
||||
LOG.exception(ex)
|
||||
raise NotesRetrievalError(
|
||||
"Unhandled error during retrieval of notes"
|
||||
)
|
||||
# Get the auth token once per retrieve, not once per note.
|
||||
if notes:
|
||||
auth_token = self.get_token()
|
||||
# resolve the note urls
|
||||
# TODO: threaded?
|
||||
for note in notes:
|
||||
self._resolve_note_url(note, auth_token)
|
||||
return notes
|
||||
|
||||
def _resolve_note_url(self, note, auth_token):
|
||||
"""Resolve and set the value obtained from the URL for a Note.
|
||||
|
||||
:param note: the Note object to retreive and set the value for.
|
||||
:param auth_token: the authorization token set as a header for the URL
|
||||
request if one is indicated as needed by the note.
|
||||
|
||||
If there is data retrieved at the note's url, set the
|
||||
resolved_url_value with those contents.
|
||||
|
||||
If there is no url for the note, return, with resolved_url_value as
|
||||
None
|
||||
|
||||
If there is no data retrieved, resolved_url_value for the note remains
|
||||
None
|
||||
|
||||
If there is an error related to retreiving the note's url value, the
|
||||
resolved_url_value is set to a placeholder value indicating that the
|
||||
value could not be obtained.
|
||||
"""
|
||||
if not isinstance(note, Note):
|
||||
LOG.debug(
|
||||
"Note is None or not a Note object. URL will not be resolved"
|
||||
)
|
||||
return
|
||||
|
||||
if not note.link_url:
|
||||
LOG.debug("Note %s has no link to resolve", note.note_id)
|
||||
return
|
||||
|
||||
contents = None
|
||||
try:
|
||||
headers = {}
|
||||
|
||||
# Don't pass credentials if not needed.
|
||||
if note.is_auth_link:
|
||||
headers['X-Auth-Token'] = auth_token
|
||||
|
||||
response = requests.get(
|
||||
note.link_url,
|
||||
headers=headers,
|
||||
timeout=(self.connect_timeout, self.read_timeout))
|
||||
response.raise_for_status()
|
||||
|
||||
# Set the valid response text to the note
|
||||
note.resolved_url_value = response.text
|
||||
|
||||
except HTTPError as he:
|
||||
# A bad status code - don't stop, but log and indicate in note.
|
||||
LOG.info(
|
||||
"Note %s has a url returning a bad status code: %s",
|
||||
note.note_id, response.status_code
|
||||
)
|
||||
note.resolved_url_value = (
|
||||
"Note contents could not be retrieved. URL lookup failed "
|
||||
"with status code: {}"
|
||||
).format(response.status_code)
|
||||
except RequestException as rex:
|
||||
# A more serious exception; log and indicate in the note
|
||||
LOG.exception(rex)
|
||||
note.resolved_url_value = (
|
||||
"Note contents could not be retrieved. URL lookup was unable "
|
||||
"to complete"
|
||||
)
|
||||
except Exception as ex:
|
||||
# Whatever's left, log and indicate in the note
|
||||
LOG.exception(ex)
|
||||
note.resolved_url_value = (
|
||||
"Note contents could not be retrieved due to unexpected "
|
||||
"circumstances"
|
||||
)
|
||||
|
||||
|
||||
class Note:
|
||||
"""Model object representing a note
|
||||
|
||||
:param assoc_id: arbitrary value like action/xxxxxxx or
|
||||
step/xxxxxxx/step_name, useful for lookup, set by note creator
|
||||
Limit: 128 characters
|
||||
:param subject: arbitrary value to be used as the subject of the note,
|
||||
useful to a human e.g. mtn15r11n0001, set by note creator
|
||||
Limit: 128 characters
|
||||
:param sub_type: arbitrary value used to qualify the subject of the note,
|
||||
e.g. Node, Action, Step, set by note creator
|
||||
Limit: 128 characters
|
||||
:param note_val: the text value of the note, the contents of info to be
|
||||
displayed as note
|
||||
:param link_url: optional url that should be followed when the note is
|
||||
retrieved to append to its value
|
||||
:param is_auth_link: boolean if Shipyard's service ID auth credentials are
|
||||
needed to make the call to follow the link default=false
|
||||
:param verbosity: integer, 1-5 indicating the verbosity level, default = 1
|
||||
:param note_id: ULID that uniquely represents a note. Users are not
|
||||
expected to pass a value for the ID of a note, it will be assigned
|
||||
:param note_timestamp: String representation of the timestamp for the note
|
||||
"""
|
||||
def __init__(self, assoc_id, subject, sub_type, note_val,
|
||||
verbosity=None, link_url=None, is_auth_link=None,
|
||||
note_id=None, note_timestamp=None):
|
||||
self.assoc_id = assoc_id
|
||||
self.subject = subject
|
||||
self.sub_type = sub_type
|
||||
self.note_val = note_val
|
||||
self.verbosity = verbosity or MIN_VERBOSITY
|
||||
self.link_url = link_url
|
||||
self.is_auth_link = is_auth_link or False
|
||||
self.note_id = note_id or ulid.ulid()
|
||||
self.note_timestamp = note_timestamp or str(datetime.utcnow())
|
||||
self._resolved_url_value = None
|
||||
|
||||
@property
|
||||
def resolved_url_value(self):
|
||||
return self._resolved_url_value
|
||||
|
||||
@resolved_url_value.setter
|
||||
def resolved_url_value(self, value):
|
||||
self._resolved_url_value = value
|
||||
|
||||
def view(self):
|
||||
"""Returns the user-facing dictionary version of the Note"""
|
||||
return {
|
||||
'assoc_id': self.assoc_id,
|
||||
'subject': self.subject,
|
||||
'sub_type': self.sub_type,
|
||||
'note_val': self.note_val,
|
||||
'verbosity': self.verbosity,
|
||||
'note_id': self.note_id,
|
||||
'note_timestamp': self.note_timestamp,
|
||||
'resolved_url_value': self.resolved_url_value,
|
||||
}
|
||||
|
||||
|
||||
class Query:
|
||||
"""Model object for a query to retrieve notes
|
||||
|
||||
:param assoc_id_pattern: The pattern to match to the assoc_id for a note.
|
||||
:param max_verbosity: optional integer 1-5, defaults to 5 (everything)
|
||||
:param exact_match: boolean, defaults to False. If true, the
|
||||
assoc_id_pattern will be used precisely, otherwise assoc_id_pattern
|
||||
will be matched to the start of the assoc_id for notes.
|
||||
"""
|
||||
def __init__(self, assoc_id_pattern, max_verbosity=None, exact_match=None):
|
||||
self.assoc_id_pattern = assoc_id_pattern
|
||||
self.max_verbosity = max_verbosity or MAX_VERBOSITY
|
||||
self.exact_match = exact_match or False
|
||||
|
||||
|
||||
class NotesStorage(metaclass=abc.ABCMeta):
|
||||
"""NotesStorage abstract base class
|
||||
|
||||
Defines the interface for NotesStorage implementations that provide the
|
||||
specific mappings of Note objects to the target data store.
|
||||
"""
|
||||
@abc.abstractmethod
|
||||
def store(self, note):
|
||||
"""Store a Note object, return the note object as stored
|
||||
|
||||
:param note: a Note object
|
||||
:returns: A single Note object, as was persisted.
|
||||
:raises NotesStorageError: When there is a failure to create the note.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def retrieve(self, query):
|
||||
"""Query for a list of Note objects
|
||||
|
||||
:param query: a Notes Query object representing the notes to be
|
||||
retrieved.
|
||||
:returns: List of Note objects matching the query
|
||||
:raises NotesRetrievalError: when there is a failure to retrieve notes,
|
||||
however an empty list is expected to be returned in the case of no
|
||||
results.
|
||||
"""
|
||||
pass
|
@ -0,0 +1,159 @@
|
||||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
import logging
|
||||
|
||||
from .notes import MAX_VERBOSITY
|
||||
from .notes import MIN_VERBOSITY
|
||||
from .notes import Query
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
# Constants and magic numbers for actions:
|
||||
# [7:33] to slice a string like:
|
||||
# action/12345678901234567890123456
|
||||
# matching the patterns in this helper.
|
||||
ACTION_KEY_PATTERN = "action/{}"
|
||||
ACTION_LOOKUP_PATTERN = "action/"
|
||||
ACTION_ID_START = 7
|
||||
ACTION_ID_END = 33
|
||||
|
||||
|
||||
class NotesHelper:
|
||||
"""Notes Helper
|
||||
|
||||
Provides helper methods for the common use cases for notes
|
||||
:param notes_manager: the NotesManager object to use
|
||||
"""
|
||||
def __init__(self, notes_manager):
|
||||
self.nm = notes_manager
|
||||
|
||||
def _failsafe_make_note(self, assoc_id, subject, sub_type, note_val,
|
||||
verbosity=None, link_url=None, is_auth_link=None):
|
||||
"""LOG and continue on any note creation failure"""
|
||||
try:
|
||||
self.nm.create(
|
||||
assoc_id=assoc_id,
|
||||
subject=subject,
|
||||
sub_type=sub_type,
|
||||
note_val=note_val,
|
||||
verbosity=verbosity,
|
||||
link_url=link_url,
|
||||
is_auth_link=is_auth_link,
|
||||
)
|
||||
except Exception as ex:
|
||||
LOG.warn(
|
||||
"Creating note for {} encountered a problem, exception info "
|
||||
"follows, but processing is not halted for notes.",
|
||||
assoc_id
|
||||
)
|
||||
LOG.exception(ex)
|
||||
|
||||
def _failsafe_get_notes(self, assoc_id_pattern, max_verbosity,
|
||||
exact_match):
|
||||
"""LOG and continue on any note retrieval failure"""
|
||||
try:
|
||||
q = Query(assoc_id_pattern, max_verbosity, exact_match)
|
||||
return self.nm.retrieve(q)
|
||||
except Exception as ex:
|
||||
LOG.warn(
|
||||
"Note retrieval for {} encountered a problem, exception "
|
||||
"info follows, but processing is not halted for notes.",
|
||||
assoc_id_pattern
|
||||
)
|
||||
LOG.exception(ex)
|
||||
return []
|
||||
|
||||
def make_action_note(self, action_id, note_val, subject=None,
|
||||
sub_type=None, verbosity=None, link_url=None,
|
||||
is_auth_link=None):
|
||||
"""Creates an action note using a convention for the note's assoc_id
|
||||
|
||||
:param action_id: the ULID id of an action
|
||||
:param note_val: the text for the note
|
||||
:param subject: optional subject for the note. Defaults to the
|
||||
action_id
|
||||
:param sub_type: optional subject type for the note, defaults to
|
||||
"action metadata"
|
||||
:param verbosity: optional verbosity for the note, defaults to 1,
|
||||
i.e.: summary level
|
||||
:param link_url: optional link URL if there's additional information
|
||||
to retreive from another source.
|
||||
:param is_auth_link: optional, defaults to False, indicating if there
|
||||
is a need to send a Shipyard service account token with the
|
||||
request to the optional URL
|
||||
"""
|
||||
assoc_id = ACTION_KEY_PATTERN.format(action_id)
|
||||
if subject is None:
|
||||
subject = action_id
|
||||
if sub_type is None:
|
||||
sub_type = "action metadata"
|
||||
if verbosity is None:
|
||||
verbosity = 1
|
||||
|
||||
self._failsafe_make_note(
|
||||
assoc_id=assoc_id,
|
||||
subject=subject,
|
||||
sub_type=sub_type,
|
||||
note_val=note_val,
|
||||
verbosity=verbosity,
|
||||
link_url=link_url,
|
||||
is_auth_link=is_auth_link,
|
||||
)
|
||||
|
||||
def get_all_action_notes(self, verbosity=None):
|
||||
"""Retrieve notes for all actions, in a dictionary keyed by action id.
|
||||
|
||||
:param verbosity: optional, 1-5, the maximum verbosity level to
|
||||
retrieve, defaults to 1 (most summary level)
|
||||
|
||||
Warning: if there are a lot of URL links in notes, this could take a
|
||||
long time. The default verbosity of 1 attempts to avoid this as there
|
||||
is less expectation of URL links on summary notes.
|
||||
"""
|
||||
max_verbosity = verbosity or MIN_VERBOSITY
|
||||
notes = self._failsafe_get_notes(
|
||||
assoc_id_pattern=ACTION_LOOKUP_PATTERN,
|
||||
max_verbosity=verbosity,
|
||||
exact_match=False
|
||||
)
|
||||
note_dict = {}
|
||||
for n in notes:
|
||||
# magic numbers [7:33] to slice a string like:
|
||||
# action/12345678901234567890123456/something
|
||||
# matching the convention in this helper.
|
||||
# in the case where there are non-compliant, the slice will make
|
||||
# the action_id a garbage key and that note will not be easily
|
||||
# associated.
|
||||
action_id = n.assoc_id[ACTION_ID_START:ACTION_ID_END]
|
||||
if action_id not in note_dict:
|
||||
note_dict[action_id] = []
|
||||
note_dict[action_id].append(n)
|
||||
return note_dict
|
||||
|
||||
def get_action_notes(self, action_id, verbosity=None):
|
||||
"""Retrive notes related to a particular action
|
||||
|
||||
:param action_id: the action for which to retrieve notes.
|
||||
:param verbosity: optional, 1-5, the maximum verbosity level to
|
||||
retrieve, defaults to 5 (most detailed level)
|
||||
"""
|
||||
assoc_id_pattern = ACTION_KEY_PATTERN.format(action_id)
|
||||
max_verbosity = verbosity or MAX_VERBOSITY
|
||||
exact_match = True
|
||||
return self._failsafe_get_notes(
|
||||
assoc_id_pattern=assoc_id_pattern,
|
||||
max_verbosity=max_verbosity,
|
||||
exact_match=exact_match
|
||||
)
|
@ -0,0 +1,167 @@
|
||||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
"""ShipyardSQLNotesStorage
|
||||
|
||||
Implementation of NotesStorage that is based on the structure of the notes
|
||||
table as defined in Shipyard. Accepts a SQLAlchemy engine as input, and
|
||||
generates a model class from the notes table.
|
||||
|
||||
Mapping to/from Note objects is encapsulated here for a consistent interface
|
||||
"""
|
||||
from contextlib import contextmanager
|
||||
import logging
|
||||
|
||||
from sqlalchemy import and_
|
||||
from sqlalchemy import Column
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy import Text
|
||||
from sqlalchemy import types
|
||||
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from .notes import Note
|
||||
from .notes import NotesStorage
|
||||
from .errors import NotesError
|
||||
from .errors import NotesInitializationError
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
Base = declarative_base()
|
||||
|
||||
|
||||
class TNote(Base):
|
||||
"""Notes ORM class"""
|
||||
__tablename__ = 'notes'
|
||||
|
||||
# These must align with the table defined using Alembic
|
||||
note_id = Column('note_id', types.String(26), primary_key=True)
|
||||
assoc_id = Column('assoc_id', types.String(128), nullable=False)
|
||||
subject = Column('subject', types.String(128), nullable=False)
|
||||
sub_type = Column('sub_type', types.String(128), nullable=False)
|
||||
note_val = Column('note_val', Text, nullable=False)
|
||||
verbosity = Column('verbosity', types.Integer, nullable=False)
|
||||
link_url = Column('link_url', Text, nullable=True)
|
||||
is_auth_link = Column('is_auth_link', types.Boolean, nullable=False)
|
||||
note_timestamp = Column('note_timestamp',
|
||||
types.TIMESTAMP(timezone=True),
|
||||
server_default=func.now())
|
||||
|
||||
|
||||
class ShipyardSQLNotesStorage(NotesStorage):
|
||||
"""SQL Alchemy implementation of a notes storage; Shipayrd table structure
|
||||
|
||||
Accepts a SQL Alchemy Engine to serve as the connection to the persistence
|
||||
layer for Notes.
|
||||
|
||||
:param engine_getter: A method that can be used to get SQLAlchemy engine
|
||||
to use
|
||||
"""
|
||||
def __init__(self, engine_getter):
|
||||
try:
|
||||
self._engine_getter = engine_getter
|
||||
self._session = None
|
||||
except Exception as ex:
|
||||
LOG.exception(ex)
|
||||
raise NotesInitializationError(
|
||||
"Misconfiguration has casuse a failure to setup the desired "
|
||||
"database connection for Notes."
|
||||
)
|
||||
|
||||
def _get_session(self):
|
||||
"""Lazy initilize the sessionmaker, invoke the engine getter, and
|
||||
use it to return a session
|
||||
"""
|
||||
if not self._session:
|
||||
self._session = sessionmaker(bind=self._engine_getter())
|
||||
return self._session()
|
||||
|
||||
@contextmanager
|
||||
def session_scope(self):
|
||||
"""Context manager for a SQLAlchemy session"""
|
||||
session = self._get_session()
|
||||
try:
|
||||
yield session
|
||||
session.commit()
|
||||
except Exception as ex:
|
||||
session.rollback()
|
||||
if isinstance(ex, NotesError):
|
||||
raise
|
||||
else:
|
||||
LOG.exception(ex)
|
||||
raise NotesError(
|
||||
"An unexpected error has occurred while attempting to "
|
||||
"interact with the database for note storage"
|
||||
)
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
def store(self, note):
|
||||
"""Store a note in the database"""
|
||||
r_note = None
|
||||
with self.session_scope() as session:
|
||||
tnote = self._map(note, TNote)
|
||||
session.add(tnote)
|
||||
r_note = self._map(tnote, Note)
|
||||
return r_note
|
||||
|
||||
def retrieve(self, query):
|
||||
a_id_pat = query.assoc_id_pattern
|
||||
m_verb = query.max_verbosity
|
||||
r_notes = []
|
||||
with self.session_scope() as session:
|
||||
notes_res = []
|
||||
if (query.exact_match):
|
||||
n_qry = session.query(TNote).filter(
|
||||
and_(
|
||||
TNote.assoc_id == a_id_pat,
|
||||
TNote.verbosity <= m_verb
|
||||
)
|
||||
)
|
||||
else:
|
||||
n_qry = session.query(TNote).filter(
|
||||
and_(
|
||||
TNote.assoc_id.like(a_id_pat + '%'),
|
||||
TNote.verbosity <= m_verb
|
||||
)
|
||||
)
|
||||
db_notes = n_qry.all()
|
||||
for tn in db_notes:
|
||||
r_notes.append(self._map(tn, Note))
|
||||
return r_notes
|
||||
|
||||
def _map(self, src, target_type):
|
||||
"""Maps a Note object to/from a TNote object.
|
||||
|
||||
:param src: the object to use as a source
|
||||
:param target_type: the type of object to create and map to
|
||||
"""
|
||||
try:
|
||||
tgt = target_type(
|
||||
assoc_id=src.assoc_id,
|
||||
subject=src.subject,
|
||||
sub_type=src.sub_type,
|
||||
note_val=src.note_val,
|
||||
verbosity=src.verbosity,
|
||||
link_url=src.link_url,
|
||||
is_auth_link=src.is_auth_link,
|
||||
note_id=src.note_id,
|
||||
note_timestamp=src.note_timestamp
|
||||
)
|
||||
except AttributeError as ae:
|
||||
LOG.exception(ae)
|
||||
raise NotesError(
|
||||
"Note could not be translated from/to SQL form; mapping error"
|
||||
)
|
||||
return tgt
|
@ -0,0 +1,43 @@
|
||||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
"""Implementations of the NotesStorage base class"""
|
||||
from .notes import NotesStorage
|
||||
|
||||
|
||||
class MemoryNotesStorage(NotesStorage):
|
||||
"""In-memory note storage
|
||||
|
||||
Primarily useful for testing
|
||||
"""
|
||||
def __init__(self):
|
||||
self.storage = {}
|
||||
|
||||
def store(self, note):
|
||||
self.storage[note.note_id] = note
|
||||
return note
|
||||
|
||||
def retrieve(self, query):
|
||||
pat = query.assoc_id_pattern
|
||||
mv = query.max_verbosity
|
||||
notes = []
|
||||
if query.exact_match:
|
||||
for note in self.storage.values():
|
||||
if note.assoc_id == pat and note.verbosity <= mv:
|
||||
notes.append(note)
|
||||
else:
|
||||
for note in self.storage.values():
|
||||
if note.assoc_id.startswith(pat) and note.verbosity <= mv:
|
||||
notes.append(note)
|
||||
return notes
|
@ -228,6 +228,17 @@ SECTIONS = [
|
||||
default=300,
|
||||
help='Airship component validation timeout (in seconds)'
|
||||
),
|
||||
cfg.IntOpt(
|
||||
'notes_connect_timeout',
|
||||
default=5,
|
||||
help=('Maximum time to wait to connect to a note source URL '
|
||||
'(in seconds)')
|
||||
),
|
||||
cfg.IntOpt(
|
||||
'notes_read_timeout',
|
||||
default=10,
|
||||
help='Read timeout for a note source URL (in seconds)'
|
||||
),
|
||||
]
|
||||
),
|
||||
ConfigSection(
|
||||
|
@ -32,6 +32,7 @@ from shipyard_airflow.control.base import BaseResource
|
||||
from shipyard_airflow.control.helpers import configdocs_helper
|
||||
from shipyard_airflow.control.helpers.configdocs_helper import (
|
||||
ConfigdocsHelper)
|
||||
from shipyard_airflow.control.helpers.notes import NOTES as notes_helper
|
||||
from shipyard_airflow.control.json_schemas import ACTION
|
||||
from shipyard_airflow.db.db import AIRFLOW_DB, SHIPYARD_DB
|
||||
from shipyard_airflow.errors import ApiError
|
||||
@ -182,6 +183,10 @@ class ActionsResource(BaseResource):
|
||||
|
||||
# insert the action into the shipyard db
|
||||
self.insert_action(action=action)
|
||||
notes_helper.make_action_note(
|
||||
action_id=action['id'],
|
||||
note_val="Configdoc revision {}".format(action['committed_rev_id'])
|
||||
)
|
||||
self.audit_control_command_db({
|
||||
'id': ulid.ulid(),
|
||||
'action_id': action['id'],
|
||||
@ -202,6 +207,7 @@ class ActionsResource(BaseResource):
|
||||
all_dag_runs = self.get_dag_run_map()
|
||||
all_tasks = self.get_all_tasks_db()
|
||||
|
||||
notes = notes_helper.get_all_action_notes(verbosity=1)
|
||||
# correlate the actions and dags into a list of action entites
|
||||
actions = []
|
||||
|
||||
@ -222,6 +228,9 @@ class ActionsResource(BaseResource):
|
||||
'%Y-%m-%dT%H:%M:%S') == dag_key_date
|
||||
]
|
||||
action['steps'] = format_action_steps(action_id, action_tasks)
|
||||
action['notes'] = []
|
||||
for note in notes.get(action_id, []):
|
||||
action['notes'].append(note.view())
|
||||
actions.append(action)
|
||||
|
||||
return actions
|
||||
|
@ -0,0 +1,49 @@
|
||||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
"""Notes
|
||||
|
||||
Provides setup and access to a NotesHelper object that is used across the
|
||||
API
|
||||
"""
|
||||
import logging
|
||||
|
||||
from oslo_config import cfg
|
||||
|
||||
from shipyard_airflow.common.notes.notes import NotesManager
|
||||
from shipyard_airflow.common.notes.notes_helper import NotesHelper
|
||||
from shipyard_airflow.common.notes.storage_impl_db import (
|
||||
ShipyardSQLNotesStorage
|
||||
)
|
||||
from shipyard_airflow.control.service_endpoints import get_token
|
||||
from shipyard_airflow.db.db import SHIPYARD_DB
|
||||
|
||||
CONF = cfg.CONF
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _notes_manager():
|
||||
"""Setup a NotesManager object using Shipyard settings"""
|
||||
sy_engine_getter = SHIPYARD_DB.get_engine
|
||||
return NotesManager(
|
||||
ShipyardSQLNotesStorage(sy_engine_getter),
|
||||
get_token,
|
||||
CONF.requests_config.notes_connect_timeout,
|
||||
CONF.requests_config.notes_read_timeout
|
||||
)
|
||||
|
||||
|
||||
# NOTES is the notes manager that can be imported and used by other modules
|
||||
# for notes functionality across the Shipyard API.
|
||||
NOTES = NotesHelper(_notes_manager())
|
353
src/bin/shipyard_airflow/tests/unit/common/notes/test_notes.py
Normal file
353
src/bin/shipyard_airflow/tests/unit/common/notes/test_notes.py
Normal file
@ -0,0 +1,353 @@
|
||||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Tests for the Notes component"""
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
import responses
|
||||
|
||||
from shipyard_airflow.common.notes.errors import (
|
||||
NotesInitializationError,
|
||||
NotesRetrievalError,
|
||||
NotesStorageError
|
||||
)
|
||||
from shipyard_airflow.common.notes.notes import (
|
||||
Note,
|
||||
NotesManager,
|
||||
NotesStorage,
|
||||
Query
|
||||
)
|
||||
from shipyard_airflow.common.notes.storage_impl_mem import MemoryNotesStorage
|
||||
|
||||
|
||||
def get_token():
|
||||
return "token"
|
||||
|
||||
|
||||
class NotesStorageErrorImpl(NotesStorage):
|
||||
def store(self, note):
|
||||
raise Exception("Didn't see it coming")
|
||||
|
||||
def retrieve(self, query):
|
||||
raise Exception("Outta Nowhere")
|
||||
|
||||
|
||||
class NotesStorageExpectedErrorImpl(NotesStorage):
|
||||
def store(self, note):
|
||||
raise NotesStorageError("Expected")
|
||||
|
||||
def retrieve(self, query):
|
||||
raise NotesRetrievalError("Expected")
|
||||
|
||||
|
||||
class TestNotesManager:
|
||||
def test_init(self):
|
||||
with pytest.raises(NotesInitializationError) as nie:
|
||||
nm = NotesManager(None, None)
|
||||
assert "Storage object is not" in str(nie.value)
|
||||
|
||||
with pytest.raises(NotesInitializationError) as nie:
|
||||
nm = NotesManager({}, None)
|
||||
assert "Storage object is not" in str(nie.value)
|
||||
|
||||
with pytest.raises(NotesInitializationError) as nie:
|
||||
nm = NotesManager(MemoryNotesStorage(), None)
|
||||
assert "Parameter get_token" in str(nie.value)
|
||||
|
||||
with pytest.raises(NotesInitializationError) as nie:
|
||||
nm = NotesManager(MemoryNotesStorage(), {})
|
||||
assert "Parameter get_token" in str(nie.value)
|
||||
|
||||
nm = NotesManager(MemoryNotesStorage(), get_token)
|
||||
assert nm.connect_timeout == 3
|
||||
assert nm.read_timeout == 10
|
||||
|
||||
nm = NotesManager(MemoryNotesStorage(), get_token, connect_timeout=99,
|
||||
read_timeout=999)
|
||||
assert nm.connect_timeout == 99
|
||||
assert nm.read_timeout == 999
|
||||
assert isinstance(nm.storage, MemoryNotesStorage)
|
||||
|
||||
def test_store_retrieve_expected_exception_handling(self):
|
||||
nm = NotesManager(NotesStorageExpectedErrorImpl(), get_token)
|
||||
with pytest.raises(NotesStorageError) as nse:
|
||||
n = Note(
|
||||
assoc_id="test1/11111/aaa",
|
||||
subject="store_retrieve_1",
|
||||
sub_type="test",
|
||||
note_val="this is my note 1"
|
||||
)
|
||||
n.note_id = None
|
||||
nm.store(n)
|
||||
assert "Expected" == str(nse.value)
|
||||
|
||||
with pytest.raises(NotesRetrievalError) as nse:
|
||||
nm.retrieve(Query("test"))
|
||||
assert "Expected" == str(nse.value)
|
||||
|
||||
def test_store_retrieve_unexpected_exception_handling(self):
|
||||
nm = NotesManager(NotesStorageErrorImpl(), get_token)
|
||||
with pytest.raises(NotesStorageError) as nse:
|
||||
n = Note(
|
||||
assoc_id="test1/11111/aaa",
|
||||
subject="store_retrieve_1",
|
||||
sub_type="test",
|
||||
note_val="this is my note 1"
|
||||
)
|
||||
n.note_id = None
|
||||
nm.store(n)
|
||||
assert "Unhandled" in str(nse.value)
|
||||
|
||||
with pytest.raises(NotesRetrievalError) as nse:
|
||||
nm.retrieve(Query("test"))
|
||||
assert "Unhandled" in str(nse.value)
|
||||
|
||||
def test_store_retrieve_basic(self):
|
||||
nm = NotesManager(MemoryNotesStorage(), get_token)
|
||||
nm.store(Note(
|
||||
assoc_id="test1/11111/aaa",
|
||||
subject="store_retrieve_1",
|
||||
sub_type="test",
|
||||
note_val="this is my note 1"
|
||||
))
|
||||
n_list = nm.retrieve(Query("test1"))
|
||||
assert len(n_list) == 1
|
||||
assert n_list[0].subject == "store_retrieve_1"
|
||||
n_list = nm.retrieve(Query("test2"))
|
||||
assert len(n_list) == 0
|
||||
|
||||
def test_store_retrieve_basic_verbosity(self):
|
||||
nm = NotesManager(MemoryNotesStorage(), get_token)
|
||||
nm.store(Note(
|
||||
assoc_id="test1/11111/aaa",
|
||||
subject="store_retrieve_1",
|
||||
sub_type="test",
|
||||
note_val="this is my note 1",
|
||||
verbosity=5
|
||||
))
|
||||
n_list = nm.retrieve(Query("test1", max_verbosity=3))
|
||||
assert len(n_list) == 0
|
||||
|
||||
def test_store_bad_verbosity(self):
|
||||
nm = NotesManager(MemoryNotesStorage(), get_token)
|
||||
with pytest.raises(NotesStorageError) as nse:
|
||||
nm.store(Note(
|
||||
assoc_id="test1/11111/aaa",
|
||||
subject="store_retrieve_1",
|
||||
sub_type="test",
|
||||
note_val="this is my note 1",
|
||||
verbosity=-4
|
||||
))
|
||||
assert "Verbosity of notes must" in str(nse.value)
|
||||
|
||||
with pytest.raises(NotesStorageError) as nse:
|
||||
nm.store(Note(
|
||||
assoc_id="test1/11111/aaa",
|
||||
subject="store_retrieve_1",
|
||||
sub_type="test",
|
||||
note_val="this is my note 1",
|
||||
verbosity=6
|
||||
))
|
||||
assert "Verbosity of notes must" in str(nse.value)
|
||||
|
||||
def test_store_retrieve_multi(self):
|
||||
nm = NotesManager(MemoryNotesStorage(), get_token)
|
||||
nm.store(Note(
|
||||
assoc_id="test1/11111/aaa",
|
||||
subject="store_retrieve",
|
||||
sub_type="test",
|
||||
note_val="this is my note 1"
|
||||
))
|
||||
nm.store(Note(
|
||||
assoc_id="test1/11111/bbb",
|
||||
subject="store_retrieve",
|
||||
sub_type="test",
|
||||
note_val="this is my note 2"
|
||||
))
|
||||
nm.store(Note(
|
||||
assoc_id="test2/2222/aaa",
|
||||
subject="store_retrieve_2",
|
||||
sub_type="test",
|
||||
note_val="this is my note 3"
|
||||
))
|
||||
n_list = nm.retrieve(Query("test2"))
|
||||
assert len(n_list) == 1
|
||||
assert n_list[0].subject == "store_retrieve_2"
|
||||
n_list = nm.retrieve(Query("test1"))
|
||||
assert len(n_list) == 2
|
||||
n_list = nm.retrieve(Query("test1", exact_match=True))
|
||||
assert len(n_list) == 0
|
||||
n_list = nm.retrieve(Query("test1/11111/aaa", exact_match=True))
|
||||
assert len(n_list) == 1
|
||||
|
||||
@responses.activate
|
||||
def test_store_retrieve_urls(self):
|
||||
responses.add(
|
||||
method="GET",
|
||||
url="http://test.test",
|
||||
body="Hello from testland",
|
||||
status=200,
|
||||
content_type="text/plain"
|
||||
)
|
||||
responses.add(
|
||||
method="GET",
|
||||
url="http://test.test2",
|
||||
body="Hello from testland2",
|
||||
status=200,
|
||||
content_type="text/plain"
|
||||
)
|
||||
|
||||
nm = NotesManager(MemoryNotesStorage(), get_token)
|
||||
nm.store(Note(
|
||||
assoc_id="test1/11111/aaa",
|
||||
subject="store_retrieve3",
|
||||
sub_type="test",
|
||||
note_val="this is my note 1",
|
||||
link_url="http://test.test/"
|
||||
))
|
||||
nm.store(Note(
|
||||
assoc_id="test1/11111/bbb",
|
||||
subject="store_retrieve3",
|
||||
sub_type="test",
|
||||
note_val="this is my note 2",
|
||||
link_url="http://test.test2/"
|
||||
))
|
||||
n_list = nm.retrieve(Query("test1"))
|
||||
assert len(n_list) == 2
|
||||
for n in n_list:
|
||||
assert n.resolved_url_value.startswith("Hello from testland")
|
||||
with pytest.raises(KeyError):
|
||||
auth_hdr = responses.calls[0].request.headers['X-Auth-Token']
|
||||
|
||||
@responses.activate
|
||||
def test_store_retrieve_url_bad_status_code(self):
|
||||
responses.add(
|
||||
method="GET",
|
||||
url="http://test.test",
|
||||
body="What note?",
|
||||
status=404,
|
||||
content_type="text/plain"
|
||||
)
|
||||
responses.add(
|
||||
method="GET",
|
||||
url="http://test.test2",
|
||||
body="Hello from testland2",
|
||||
status=200,
|
||||
content_type="text/plain"
|
||||
)
|
||||
|
||||
nm = NotesManager(MemoryNotesStorage(), get_token)
|
||||
nm.store(Note(
|
||||
assoc_id="test1/11111/aaa",
|
||||
subject="store_retrieve3",
|
||||
sub_type="test",
|
||||
note_val="this is my note 1",
|
||||
link_url="http://test.test/"
|
||||
))
|
||||
nm.store(Note(
|
||||
assoc_id="test1/11111/bbb",
|
||||
subject="store_retrieve3",
|
||||
sub_type="test",
|
||||
note_val="this is my note 2",
|
||||
link_url="http://test.test2/"
|
||||
))
|
||||
n_list = nm.retrieve(Query("test1"))
|
||||
assert len(n_list) == 2
|
||||
for n in n_list:
|
||||
if n.assoc_id == "test1/11111/aaa":
|
||||
assert "failed with status code: 404" in n.resolved_url_value
|
||||
else:
|
||||
assert n.resolved_url_value.startswith("Hello from testland")
|
||||
|
||||
@responses.activate
|
||||
def test_store_retrieve_url_does_not_exist(self):
|
||||
responses.add(
|
||||
method="GET",
|
||||
url="http://test.test2",
|
||||
body="Hello from testland2",
|
||||
status=200,
|
||||
content_type="text/plain"
|
||||
)
|
||||
|
||||
nm = NotesManager(MemoryNotesStorage(), get_token)
|
||||
nm.store(Note(
|
||||
assoc_id="test1/11111/aaa",
|
||||
subject="store_retrieve3",
|
||||
sub_type="test",
|
||||
note_val="this is my note 1",
|
||||
link_url="test_breakage://test.test/"
|
||||
))
|
||||
nm.store(Note(
|
||||
assoc_id="test1/11111/bbb",
|
||||
subject="store_retrieve3",
|
||||
sub_type="test",
|
||||
note_val="this is my note 2",
|
||||
link_url="http://test.test2/"
|
||||
))
|
||||
n_list = nm.retrieve(Query("test1"))
|
||||
assert len(n_list) == 2
|
||||
for n in n_list:
|
||||
if n.assoc_id == "test1/11111/aaa":
|
||||
assert "URL lookup was unable" in n.resolved_url_value
|
||||
else:
|
||||
assert n.resolved_url_value.startswith("Hello from testland")
|
||||
|
||||
@responses.activate
|
||||
def test_store_retrieve_with_auth(self):
|
||||
responses.add(
|
||||
method="GET",
|
||||
url="http://test.test2",
|
||||
body="Hello from testland2",
|
||||
status=200,
|
||||
content_type="text/plain"
|
||||
)
|
||||
|
||||
nm = NotesManager(MemoryNotesStorage(), get_token)
|
||||
nm.store(Note(
|
||||
assoc_id="test1/11111/bbb",
|
||||
subject="store_retrieve3",
|
||||
sub_type="test",
|
||||
note_val="this is my note 2",
|
||||
link_url="http://test.test2/",
|
||||
is_auth_link=True
|
||||
))
|
||||
n_list = nm.retrieve(Query("test1"))
|
||||
assert len(n_list) == 1
|
||||
for n in n_list:
|
||||
assert n.resolved_url_value == "Hello from testland2"
|
||||
auth_hdr = responses.calls[0].request.headers['X-Auth-Token']
|
||||
assert 'token' == auth_hdr
|
||||
|
||||
def test_note_view(self):
|
||||
nm = NotesManager(MemoryNotesStorage(), get_token)
|
||||
nm.store(Note(
|
||||
assoc_id="test1/11111/aaa",
|
||||
subject="store_retrieve_1",
|
||||
sub_type="test",
|
||||
note_val="this is my note 1"
|
||||
))
|
||||
n_list = nm.retrieve(Query("test1"))
|
||||
assert len(n_list) == 1
|
||||
nid = n_list[0].note_id
|
||||
nt = n_list[0].note_timestamp
|
||||
assert n_list[0].view() == {
|
||||
'assoc_id': 'test1/11111/aaa',
|
||||
'subject': 'store_retrieve_1',
|
||||
'sub_type': 'test',
|
||||
'note_val': 'this is my note 1',
|
||||
'verbosity': 1,
|
||||
'note_id': nid,
|
||||
'note_timestamp': nt,
|
||||
'resolved_url_value': None,
|
||||
}
|
@ -58,7 +58,6 @@ def get_doc_returner(style, ds_name):
|
||||
# if passed a name of 'defaulted' clear the section
|
||||
if ds_name == 'defaulted':
|
||||
dc.data["physical_provisioner"] = None
|
||||
print(dc.__dict__)
|
||||
return [dc]
|
||||
elif doc == 'dep-strat':
|
||||
return [strategy]
|
||||
|
@ -24,6 +24,11 @@ from oslo_config import cfg
|
||||
import pytest
|
||||
import responses
|
||||
|
||||
from shipyard_airflow.common.notes.notes import NotesManager
|
||||
from shipyard_airflow.common.notes.notes_helper import NotesHelper
|
||||
from shipyard_airflow.common.notes.storage_impl_mem import (
|
||||
MemoryNotesStorage
|
||||
)
|
||||
from shipyard_airflow.control.action import actions_api
|
||||
from shipyard_airflow.control.action.actions_api import ActionsResource
|
||||
from shipyard_airflow.control.base import ShipyardRequestContext
|
||||
@ -43,6 +48,15 @@ CONF = cfg.CONF
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_token():
|
||||
"""Stub method to use for NotesHelper/NotesManager"""
|
||||
return "token"
|
||||
|
||||
# Notes helper that can be mocked into various objects to prevent database
|
||||
# dependencies
|
||||
nh = NotesHelper(NotesManager(MemoryNotesStorage(), get_token))
|
||||
|
||||
|
||||
def create_req(ctx, body):
|
||||
'''creates a falcon request'''
|
||||
env = testing.create_environ(
|
||||
@ -283,8 +297,6 @@ def test_get_all_actions():
|
||||
action_resource.get_all_actions_db = actions_db
|
||||
action_resource.get_all_dag_runs_db = dag_runs_db
|
||||
action_resource.get_all_tasks_db = tasks_db
|
||||
os.environ['DB_CONN_AIRFLOW'] = 'nothing'
|
||||
os.environ['DB_CONN_SHIPYARD'] = 'nothing'
|
||||
result = action_resource.get_all_actions()
|
||||
assert len(result) == len(actions_db())
|
||||
for action in result:
|
||||
@ -296,6 +308,31 @@ def test_get_all_actions():
|
||||
assert action['dag_status'] == 'SUCCESS'
|
||||
|
||||
|
||||
@mock.patch('shipyard_airflow.control.action.actions_api.notes_helper',
|
||||
new=nh)
|
||||
def test_get_all_actions_notes(*args):
|
||||
"""
|
||||
Tests the main response from get all actions
|
||||
"""
|
||||
action_resource = ActionsResource()
|
||||
action_resource.get_all_actions_db = actions_db
|
||||
action_resource.get_all_dag_runs_db = dag_runs_db
|
||||
action_resource.get_all_tasks_db = tasks_db
|
||||
# inject some notes
|
||||
nh.make_action_note('aaaaaa', "hello from aaaaaa1")
|
||||
nh.make_action_note('aaaaaa', "hello from aaaaaa2")
|
||||
nh.make_action_note('bbbbbb', "hello from bbbbbb")
|
||||
|
||||
result = action_resource.get_all_actions()
|
||||
assert len(result) == len(actions_db())
|
||||
for action in result:
|
||||
if action['id'] == 'aaaaaa':
|
||||
assert len(action['notes']) == 2
|
||||
if action['id'] == 'bbbbbb':
|
||||
assert len(action['notes']) == 1
|
||||
assert action['notes'][0]['note_val'] == 'hello from bbbbbb'
|
||||
|
||||
|
||||
def _gen_action_resource_stubbed():
|
||||
# TODO(bryan-strassner): mabye subclass this instead?
|
||||
action_resource = ActionsResource()
|
||||
|
Loading…
x
Reference in New Issue
Block a user