Browse Source

Merge "Add notes common code for Shipyard"

changes/87/608087/16
Zuul 3 years ago
committed by Gerrit Code Review
parent
commit
282c5699b9
  1. 7
      doc/source/_static/shipyard.conf.sample
  2. 47
      src/bin/shipyard_airflow/alembic/versions/7486ddec1979_create_notes_table.py
  3. 7
      src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample
  4. 0
      src/bin/shipyard_airflow/shipyard_airflow/common/notes/__init__.py
  5. 44
      src/bin/shipyard_airflow/shipyard_airflow/common/notes/errors.py
  6. 318
      src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes.py
  7. 159
      src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes_helper.py
  8. 167
      src/bin/shipyard_airflow/shipyard_airflow/common/notes/storage_impl_db.py
  9. 43
      src/bin/shipyard_airflow/shipyard_airflow/common/notes/storage_impl_mem.py
  10. 11
      src/bin/shipyard_airflow/shipyard_airflow/conf/config.py
  11. 9
      src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py
  12. 49
      src/bin/shipyard_airflow/shipyard_airflow/control/helpers/notes.py
  13. 0
      src/bin/shipyard_airflow/tests/unit/common/notes/__init__.py
  14. 353
      src/bin/shipyard_airflow/tests/unit/common/notes/test_notes.py
  15. 1
      src/bin/shipyard_airflow/tests/unit/control/test_action_validators.py
  16. 41
      src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py

7
doc/source/_static/shipyard.conf.sample

@ -369,6 +369,13 @@
# Airship component validation timeout (in seconds) (integer value)
#validation_read_timeout = 300
# Maximum time to wait to connect to a note source URL (in seconds) (integer
# value)
#notes_connect_timeout = 5
# Read timeout for a note source URL (in seconds) (integer value)
#notes_read_timeout = 10
[shipyard]

47
src/bin/shipyard_airflow/alembic/versions/7486ddec1979_create_notes_table.py

@ -0,0 +1,47 @@
"""create notes table
Revision ID: 7486ddec1979
Revises: 51b92375e5c4
Create Date: 2018-10-02 16:10:08.139378
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy import (types, func)
# revision identifiers, used by Alembic.
revision = '7486ddec1979'
down_revision = '51b92375e5c4'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'notes',
# ULID key for the note
sa.Column('note_id', types.String(26), primary_key=True),
# The supplied association id used for lookup
sa.Column('assoc_id', types.String(128), nullable=False),
# The supplied subject of the note (what is this note about?)
sa.Column('subject', types.String(128), nullable=False),
# The supplied type of the subject (what kind of thing is the subject?)
sa.Column('sub_type', types.String(128), nullable=False),
# The text value of the note
sa.Column('note_val', sa.Text, nullable=False),
# The numeric verbosity level of the note (1-5)
sa.Column('verbosity', types.Integer, nullable=False),
# An optional URL containing more info for the note
sa.Column('link_url', sa.Text, nullable=True),
# Boolean if the link requires a X-Auth-Token header
sa.Column('is_auth_link', types.Boolean, nullable=False),
# The creation timestamp for the note
sa.Column('note_timestamp',
types.TIMESTAMP(timezone=True),
server_default=func.now()),
)
def downgrade():
op.drop_table('notes')

7
src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample

@ -378,6 +378,13 @@
# Airship component validation timeout (in seconds) (integer value)
#validation_read_timeout = 300
# Maximum time to wait to connect to a note source URL (in seconds) (integer
# value)
#notes_connect_timeout = 5
# Read timeout for a note source URL (in seconds) (integer value)
#notes_read_timeout = 10
[shipyard]

0
src/bin/shipyard_airflow/shipyard_airflow/common/notes/__init__.py

44
src/bin/shipyard_airflow/shipyard_airflow/common/notes/errors.py

@ -0,0 +1,44 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Errors for the Notes component"""
class NotesError(Exception):
"""Base exception for all NotesErrors"""
pass
class NotesInitializationError(NotesError):
"""NotesInitializationError
Raised for errors while initializing a Notes instance
"""
pass
class NotesRetrievalError(NotesError):
"""NotesRetrievalError
Raised when there is an error retrieving notes
"""
pass
class NotesStorageError(NotesError):
"""NotesStorageError
Raised when there is an error attempting to store a note.
"""
pass

318
src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes.py

@ -0,0 +1,318 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Notes
A reusable component allowing for recording and retreiving information related
to arbitrarily useage-based keys (the usage makes the association keys to their
string-based liking). The intention for Notes is generally for additional
non-fielded information that may be of interest to a user. Notes are not
intended to store info that would drive code paths or decisions. This is not
an arbitrary use key-value database.
"""
import abc
from datetime import datetime
import logging
import requests
from requests.exceptions import HTTPError
from requests.exceptions import RequestException
import ulid
from .errors import NotesInitializationError
from .errors import NotesRetrievalError
from .errors import NotesStorageError
LOG = logging.getLogger(__name__)
MAX_VERBOSITY = 5
MIN_VERBOSITY = 1
class NotesManager:
"""Interface to store and retrieve notes
:param storage: A NotesStorage object to store and retrieve notes for a
specific storage mechanism. e.g. Database, Service
:param get_token: A method that returns an auth token that will be used
as the X-Auth-Token header when resolving url-based notes
:param connect_timeout: optional, The maximum time waiting to connect to
a URL. Defaults to 3 seconds
:param read_timeout: optional, The maximum time waiting to read the info
from a URL. Defaults to 10 seconds
Example usage:
nm = NotesManager(SQLNotesStorage("connection_info"), get_url)
a_note = nm.store(Note(...params...))
notes = list(nm.retrieve(Query("some/id")))
"""
def __init__(self, storage, get_token, connect_timeout=None,
read_timeout=None):
if not isinstance(storage, NotesStorage):
raise NotesInitializationError(
"Storage object is not suitable for use with Notes"
)
self.storage = storage
LOG.info(
"Initializing Notes with storage mechanism: %s",
storage.__class__.__name__
)
if not callable(get_token):
raise NotesInitializationError(
"Parameter get_token is not suitable for use with Notes. "
"Must be a callable."
)
self.get_token = get_token
# connect and read timeouts default to 3 and 10 seconds
self.connect_timeout = connect_timeout or 3
self.read_timeout = read_timeout or 10
def create(self, assoc_id, subject, sub_type, note_val,
verbosity=None, link_url=None, is_auth_link=None,
note_id=None, note_timestamp=None, store=True):
"""Creates and stores a Note object from parameters
Passthrough helper method to avoid additional imports for the Note
class. Most of the parameters match that of the Note constructor.
See: func:`notes.Note.__init__`
Additional Parameters:
:param store: optinal, default=True, invoke the store method
immediately upon creation, if true
"""
n = Note(assoc_id, subject, sub_type, note_val,
verbosity=None, link_url=None, is_auth_link=None,
note_id=None, note_timestamp=None)
if store:
return self.store(n)
else:
return n
def store(self, note):
"""Store a note
:param note: A Note object to store
:returns: The note, as it was after storage
"""
if note.verbosity < MIN_VERBOSITY or note.verbosity > MAX_VERBOSITY:
raise NotesStorageError(
"Verbosity of notes must range from {} "
"to {} (most verbose)".format(MIN_VERBOSITY, MAX_VERBOSITY))
try:
return self.storage.store(note)
except NotesStorageError:
raise
except Exception as ex:
LOG.exception(ex)
raise NotesStorageError("Unhandled error during storage of a note")
def retrieve(self, query):
"""Retrieve a list of notes
:param query: a query object to retrieve notes
"""
try:
notes = list(self.storage.retrieve(query))
except NotesRetrievalError:
raise
except Exception as ex:
LOG.exception(ex)
raise NotesRetrievalError(
"Unhandled error during retrieval of notes"
)
# Get the auth token once per retrieve, not once per note.
if notes:
auth_token = self.get_token()
# resolve the note urls
# TODO: threaded?
for note in notes:
self._resolve_note_url(note, auth_token)
return notes
def _resolve_note_url(self, note, auth_token):
"""Resolve and set the value obtained from the URL for a Note.
:param note: the Note object to retreive and set the value for.
:param auth_token: the authorization token set as a header for the URL
request if one is indicated as needed by the note.
If there is data retrieved at the note's url, set the
resolved_url_value with those contents.
If there is no url for the note, return, with resolved_url_value as
None
If there is no data retrieved, resolved_url_value for the note remains
None
If there is an error related to retreiving the note's url value, the
resolved_url_value is set to a placeholder value indicating that the
value could not be obtained.
"""
if not isinstance(note, Note):
LOG.debug(
"Note is None or not a Note object. URL will not be resolved"
)
return
if not note.link_url:
LOG.debug("Note %s has no link to resolve", note.note_id)
return
contents = None
try:
headers = {}
# Don't pass credentials if not needed.
if note.is_auth_link:
headers['X-Auth-Token'] = auth_token
response = requests.get(
note.link_url,
headers=headers,
timeout=(self.connect_timeout, self.read_timeout))
response.raise_for_status()
# Set the valid response text to the note
note.resolved_url_value = response.text
except HTTPError as he:
# A bad status code - don't stop, but log and indicate in note.
LOG.info(
"Note %s has a url returning a bad status code: %s",
note.note_id, response.status_code
)
note.resolved_url_value = (
"Note contents could not be retrieved. URL lookup failed "
"with status code: {}"
).format(response.status_code)
except RequestException as rex:
# A more serious exception; log and indicate in the note
LOG.exception(rex)
note.resolved_url_value = (
"Note contents could not be retrieved. URL lookup was unable "
"to complete"
)
except Exception as ex:
# Whatever's left, log and indicate in the note
LOG.exception(ex)
note.resolved_url_value = (
"Note contents could not be retrieved due to unexpected "
"circumstances"
)
class Note:
"""Model object representing a note
:param assoc_id: arbitrary value like action/xxxxxxx or
step/xxxxxxx/step_name, useful for lookup, set by note creator
Limit: 128 characters
:param subject: arbitrary value to be used as the subject of the note,
useful to a human e.g. mtn15r11n0001, set by note creator
Limit: 128 characters
:param sub_type: arbitrary value used to qualify the subject of the note,
e.g. Node, Action, Step, set by note creator
Limit: 128 characters
:param note_val: the text value of the note, the contents of info to be
displayed as note
:param link_url: optional url that should be followed when the note is
retrieved to append to its value
:param is_auth_link: boolean if Shipyard's service ID auth credentials are
needed to make the call to follow the link default=false
:param verbosity: integer, 1-5 indicating the verbosity level, default = 1
:param note_id: ULID that uniquely represents a note. Users are not
expected to pass a value for the ID of a note, it will be assigned
:param note_timestamp: String representation of the timestamp for the note
"""
def __init__(self, assoc_id, subject, sub_type, note_val,
verbosity=None, link_url=None, is_auth_link=None,
note_id=None, note_timestamp=None):
self.assoc_id = assoc_id
self.subject = subject
self.sub_type = sub_type
self.note_val = note_val
self.verbosity = verbosity or MIN_VERBOSITY
self.link_url = link_url
self.is_auth_link = is_auth_link or False
self.note_id = note_id or ulid.ulid()
self.note_timestamp = note_timestamp or str(datetime.utcnow())
self._resolved_url_value = None
@property
def resolved_url_value(self):
return self._resolved_url_value
@resolved_url_value.setter
def resolved_url_value(self, value):
self._resolved_url_value = value
def view(self):
"""Returns the user-facing dictionary version of the Note"""
return {
'assoc_id': self.assoc_id,
'subject': self.subject,
'sub_type': self.sub_type,
'note_val': self.note_val,
'verbosity': self.verbosity,
'note_id': self.note_id,
'note_timestamp': self.note_timestamp,
'resolved_url_value': self.resolved_url_value,
}
class Query:
"""Model object for a query to retrieve notes
:param assoc_id_pattern: The pattern to match to the assoc_id for a note.
:param max_verbosity: optional integer 1-5, defaults to 5 (everything)
:param exact_match: boolean, defaults to False. If true, the
assoc_id_pattern will be used precisely, otherwise assoc_id_pattern
will be matched to the start of the assoc_id for notes.
"""
def __init__(self, assoc_id_pattern, max_verbosity=None, exact_match=None):
self.assoc_id_pattern = assoc_id_pattern
self.max_verbosity = max_verbosity or MAX_VERBOSITY
self.exact_match = exact_match or False
class NotesStorage(metaclass=abc.ABCMeta):
"""NotesStorage abstract base class
Defines the interface for NotesStorage implementations that provide the
specific mappings of Note objects to the target data store.
"""
@abc.abstractmethod
def store(self, note):
"""Store a Note object, return the note object as stored
:param note: a Note object
:returns: A single Note object, as was persisted.
:raises NotesStorageError: When there is a failure to create the note.
"""
pass
@abc.abstractmethod
def retrieve(self, query):
"""Query for a list of Note objects
:param query: a Notes Query object representing the notes to be
retrieved.
:returns: List of Note objects matching the query
:raises NotesRetrievalError: when there is a failure to retrieve notes,
however an empty list is expected to be returned in the case of no
results.
"""
pass

159
src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes_helper.py

@ -0,0 +1,159 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import logging
from .notes import MAX_VERBOSITY
from .notes import MIN_VERBOSITY
from .notes import Query
LOG = logging.getLogger(__name__)
# Constants and magic numbers for actions:
# [7:33] to slice a string like:
# action/12345678901234567890123456
# matching the patterns in this helper.
ACTION_KEY_PATTERN = "action/{}"
ACTION_LOOKUP_PATTERN = "action/"
ACTION_ID_START = 7
ACTION_ID_END = 33
class NotesHelper:
"""Notes Helper
Provides helper methods for the common use cases for notes
:param notes_manager: the NotesManager object to use
"""
def __init__(self, notes_manager):
self.nm = notes_manager
def _failsafe_make_note(self, assoc_id, subject, sub_type, note_val,
verbosity=None, link_url=None, is_auth_link=None):
"""LOG and continue on any note creation failure"""
try:
self.nm.create(
assoc_id=assoc_id,
subject=subject,
sub_type=sub_type,
note_val=note_val,
verbosity=verbosity,
link_url=link_url,
is_auth_link=is_auth_link,
)
except Exception as ex:
LOG.warn(
"Creating note for {} encountered a problem, exception info "
"follows, but processing is not halted for notes.",
assoc_id
)
LOG.exception(ex)
def _failsafe_get_notes(self, assoc_id_pattern, max_verbosity,
exact_match):
"""LOG and continue on any note retrieval failure"""
try:
q = Query(assoc_id_pattern, max_verbosity, exact_match)
return self.nm.retrieve(q)
except Exception as ex:
LOG.warn(
"Note retrieval for {} encountered a problem, exception "
"info follows, but processing is not halted for notes.",
assoc_id_pattern
)
LOG.exception(ex)
return []
def make_action_note(self, action_id, note_val, subject=None,
sub_type=None, verbosity=None, link_url=None,
is_auth_link=None):
"""Creates an action note using a convention for the note's assoc_id
:param action_id: the ULID id of an action
:param note_val: the text for the note
:param subject: optional subject for the note. Defaults to the
action_id
:param sub_type: optional subject type for the note, defaults to
"action metadata"
:param verbosity: optional verbosity for the note, defaults to 1,
i.e.: summary level
:param link_url: optional link URL if there's additional information
to retreive from another source.
:param is_auth_link: optional, defaults to False, indicating if there
is a need to send a Shipyard service account token with the
request to the optional URL
"""
assoc_id = ACTION_KEY_PATTERN.format(action_id)
if subject is None:
subject = action_id
if sub_type is None:
sub_type = "action metadata"
if verbosity is None:
verbosity = 1
self._failsafe_make_note(
assoc_id=assoc_id,
subject=subject,
sub_type=sub_type,
note_val=note_val,
verbosity=verbosity,
link_url=link_url,
is_auth_link=is_auth_link,
)
def get_all_action_notes(self, verbosity=None):
"""Retrieve notes for all actions, in a dictionary keyed by action id.
:param verbosity: optional, 1-5, the maximum verbosity level to
retrieve, defaults to 1 (most summary level)
Warning: if there are a lot of URL links in notes, this could take a
long time. The default verbosity of 1 attempts to avoid this as there
is less expectation of URL links on summary notes.
"""
max_verbosity = verbosity or MIN_VERBOSITY
notes = self._failsafe_get_notes(
assoc_id_pattern=ACTION_LOOKUP_PATTERN,
max_verbosity=verbosity,
exact_match=False
)
note_dict = {}
for n in notes:
# magic numbers [7:33] to slice a string like:
# action/12345678901234567890123456/something
# matching the convention in this helper.
# in the case where there are non-compliant, the slice will make
# the action_id a garbage key and that note will not be easily
# associated.
action_id = n.assoc_id[ACTION_ID_START:ACTION_ID_END]
if action_id not in note_dict:
note_dict[action_id] = []
note_dict[action_id].append(n)
return note_dict
def get_action_notes(self, action_id, verbosity=None):
"""Retrive notes related to a particular action
:param action_id: the action for which to retrieve notes.
:param verbosity: optional, 1-5, the maximum verbosity level to
retrieve, defaults to 5 (most detailed level)
"""
assoc_id_pattern = ACTION_KEY_PATTERN.format(action_id)
max_verbosity = verbosity or MAX_VERBOSITY
exact_match = True
return self._failsafe_get_notes(
assoc_id_pattern=assoc_id_pattern,
max_verbosity=max_verbosity,
exact_match=exact_match
)

167
src/bin/shipyard_airflow/shipyard_airflow/common/notes/storage_impl_db.py

@ -0,0 +1,167 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""ShipyardSQLNotesStorage
Implementation of NotesStorage that is based on the structure of the notes
table as defined in Shipyard. Accepts a SQLAlchemy engine as input, and
generates a model class from the notes table.
Mapping to/from Note objects is encapsulated here for a consistent interface
"""
from contextlib import contextmanager
import logging
from sqlalchemy import and_
from sqlalchemy import Column
from sqlalchemy import func
from sqlalchemy import Text
from sqlalchemy import types
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from .notes import Note
from .notes import NotesStorage
from .errors import NotesError
from .errors import NotesInitializationError
LOG = logging.getLogger(__name__)
Base = declarative_base()
class TNote(Base):
"""Notes ORM class"""
__tablename__ = 'notes'
# These must align with the table defined using Alembic
note_id = Column('note_id', types.String(26), primary_key=True)
assoc_id = Column('assoc_id', types.String(128), nullable=False)
subject = Column('subject', types.String(128), nullable=False)
sub_type = Column('sub_type', types.String(128), nullable=False)
note_val = Column('note_val', Text, nullable=False)
verbosity = Column('verbosity', types.Integer, nullable=False)
link_url = Column('link_url', Text, nullable=True)
is_auth_link = Column('is_auth_link', types.Boolean, nullable=False)
note_timestamp = Column('note_timestamp',
types.TIMESTAMP(timezone=True),
server_default=func.now())
class ShipyardSQLNotesStorage(NotesStorage):
"""SQL Alchemy implementation of a notes storage; Shipayrd table structure
Accepts a SQL Alchemy Engine to serve as the connection to the persistence
layer for Notes.
:param engine_getter: A method that can be used to get SQLAlchemy engine
to use
"""
def __init__(self, engine_getter):
try:
self._engine_getter = engine_getter
self._session = None
except Exception as ex:
LOG.exception(ex)
raise NotesInitializationError(
"Misconfiguration has casuse a failure to setup the desired "
"database connection for Notes."
)
def _get_session(self):
"""Lazy initilize the sessionmaker, invoke the engine getter, and
use it to return a session
"""
if not self._session:
self._session = sessionmaker(bind=self._engine_getter())
return self._session()
@contextmanager
def session_scope(self):
"""Context manager for a SQLAlchemy session"""
session = self._get_session()
try:
yield session
session.commit()
except Exception as ex:
session.rollback()
if isinstance(ex, NotesError):
raise
else:
LOG.exception(ex)
raise NotesError(
"An unexpected error has occurred while attempting to "
"interact with the database for note storage"
)
finally:
session.close()
def store(self, note):
"""Store a note in the database"""
r_note = None
with self.session_scope() as session:
tnote = self._map(note, TNote)
session.add(tnote)
r_note = self._map(tnote, Note)
return r_note
def retrieve(self, query):
a_id_pat = query.assoc_id_pattern
m_verb = query.max_verbosity
r_notes = []
with self.session_scope() as session:
notes_res = []
if (query.exact_match):
n_qry = session.query(TNote).filter(
and_(
TNote.assoc_id == a_id_pat,
TNote.verbosity <= m_verb
)
)
else:
n_qry = session.query(TNote).filter(
and_(
TNote.assoc_id.like(a_id_pat + '%'),
TNote.verbosity <= m_verb
)
)
db_notes = n_qry.all()
for tn in db_notes:
r_notes.append(self._map(tn, Note))
return r_notes
def _map(self, src, target_type):
"""Maps a Note object to/from a TNote object.
:param src: the object to use as a source
:param target_type: the type of object to create and map to
"""
try:
tgt = target_type(
assoc_id=src.assoc_id,
subject=src.subject,
sub_type=src.sub_type,
note_val=src.note_val,
verbosity=src.verbosity,
link_url=src.link_url,
is_auth_link=src.is_auth_link,
note_id=src.note_id,
note_timestamp=src.note_timestamp
)
except AttributeError as ae:
LOG.exception(ae)
raise NotesError(
"Note could not be translated from/to SQL form; mapping error"
)
return tgt

43
src/bin/shipyard_airflow/shipyard_airflow/common/notes/storage_impl_mem.py

@ -0,0 +1,43 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Implementations of the NotesStorage base class"""
from .notes import NotesStorage
class MemoryNotesStorage(NotesStorage):
"""In-memory note storage
Primarily useful for testing
"""
def __init__(self):
self.storage = {}
def store(self, note):
self.storage[note.note_id] = note
return note
def retrieve(self, query):
pat = query.assoc_id_pattern
mv = query.max_verbosity
notes = []
if query.exact_match:
for note in self.storage.values():
if note.assoc_id == pat and note.verbosity <= mv:
notes.append(note)
else:
for note in self.storage.values():
if note.assoc_id.startswith(pat) and note.verbosity <= mv:
notes.append(note)
return notes

11
src/bin/shipyard_airflow/shipyard_airflow/conf/config.py

@ -234,6 +234,17 @@ SECTIONS = [
default=300,
help='Airship component validation timeout (in seconds)'
),
cfg.IntOpt(
'notes_connect_timeout',
default=5,
help=('Maximum time to wait to connect to a note source URL '
'(in seconds)')
),
cfg.IntOpt(
'notes_read_timeout',
default=10,
help='Read timeout for a note source URL (in seconds)'
),
]
),
ConfigSection(

9
src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py

@ -32,6 +32,7 @@ from shipyard_airflow.control.base import BaseResource
from shipyard_airflow.control.helpers import configdocs_helper
from shipyard_airflow.control.helpers.configdocs_helper import (
ConfigdocsHelper)
from shipyard_airflow.control.helpers.notes import NOTES as notes_helper
from shipyard_airflow.control.json_schemas import ACTION
from shipyard_airflow.db.db import AIRFLOW_DB, SHIPYARD_DB
from shipyard_airflow.errors import ApiError
@ -189,6 +190,10 @@ class ActionsResource(BaseResource):
# insert the action into the shipyard db
self.insert_action(action=action)
notes_helper.make_action_note(
action_id=action['id'],
note_val="Configdoc revision {}".format(action['committed_rev_id'])
)
self.audit_control_command_db({
'id': ulid.ulid(),
'action_id': action['id'],
@ -209,6 +214,7 @@ class ActionsResource(BaseResource):
all_dag_runs = self.get_dag_run_map()
all_tasks = self.get_all_tasks_db()
notes = notes_helper.get_all_action_notes(verbosity=1)
# correlate the actions and dags into a list of action entites
actions = []
@ -229,6 +235,9 @@ class ActionsResource(BaseResource):
'%Y-%m-%dT%H:%M:%S') == dag_key_date
]
action['steps'] = format_action_steps(action_id, action_tasks)
action['notes'] = []
for note in notes.get(action_id, []):
action['notes'].append(note.view())
actions.append(action)
return actions

49
src/bin/shipyard_airflow/shipyard_airflow/control/helpers/notes.py

@ -0,0 +1,49 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Notes
Provides setup and access to a NotesHelper object that is used across the
API
"""
import logging
from oslo_config import cfg
from shipyard_airflow.common.notes.notes import NotesManager
from shipyard_airflow.common.notes.notes_helper import NotesHelper
from shipyard_airflow.common.notes.storage_impl_db import (
ShipyardSQLNotesStorage
)
from shipyard_airflow.control.service_endpoints import get_token
from shipyard_airflow.db.db import SHIPYARD_DB
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
def _notes_manager():
"""Setup a NotesManager object using Shipyard settings"""
sy_engine_getter = SHIPYARD_DB.get_engine
return NotesManager(
ShipyardSQLNotesStorage(sy_engine_getter),
get_token,
CONF.requests_config.notes_connect_timeout,
CONF.requests_config.notes_read_timeout
)
# NOTES is the notes manager that can be imported and used by other modules
# for notes functionality across the Shipyard API.
NOTES = NotesHelper(_notes_manager())

0
src/bin/shipyard_airflow/tests/unit/common/notes/__init__.py

353
src/bin/shipyard_airflow/tests/unit/common/notes/test_notes.py

@ -0,0 +1,353 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for the Notes component"""
from unittest import mock
import pytest
import responses
from shipyard_airflow.common.notes.errors import (
NotesInitializationError,
NotesRetrievalError,
NotesStorageError
)
from shipyard_airflow.common.notes.notes import (
Note,
NotesManager,
NotesStorage,
Query
)
from shipyard_airflow.common.notes.storage_impl_mem import MemoryNotesStorage
def get_token():
return "token"
class NotesStorageErrorImpl(NotesStorage):
def store(self, note):
raise Exception("Didn't see it coming")
def retrieve(self, query):
raise Exception("Outta Nowhere")
class NotesStorageExpectedErrorImpl(NotesStorage):
def store(self, note):
raise NotesStorageError("Expected")
def retrieve(self, query):
raise NotesRetrievalError("Expected")
class TestNotesManager:
def test_init(self):
with pytest.raises(NotesInitializationError) as nie:
nm = NotesManager(None, None)
assert "Storage object is not" in str(nie.value)
with pytest.raises(NotesInitializationError) as nie:
nm = NotesManager({}, None)
assert "Storage object is not" in str(nie.value)
with pytest.raises(NotesInitializationError) as nie:
nm = NotesManager(MemoryNotesStorage(), None)
assert "Parameter get_token" in str(nie.value)
with pytest.raises(NotesInitializationError) as nie:
nm = NotesManager(MemoryNotesStorage(), {})
assert "Parameter get_token" in str(nie.value)
nm = NotesManager(MemoryNotesStorage(), get_token)
assert nm.connect_timeout == 3
assert nm.read_timeout == 10
nm = NotesManager(MemoryNotesStorage(), get_token, connect_timeout=99,
read_timeout=999)
assert nm.connect_timeout == 99
assert nm.read_timeout == 999
assert isinstance(nm.storage, MemoryNotesStorage)
def test_store_retrieve_expected_exception_handling(self):
nm = NotesManager(NotesStorageExpectedErrorImpl(), get_token)
with pytest.raises(NotesStorageError) as nse:
n = Note(
assoc_id="test1/11111/aaa",
subject="store_retrieve_1",
sub_type="test",
note_val="this is my note 1"
)
n.note_id = None
nm.store(n)
assert "Expected" == str(nse.value)
with pytest.raises(NotesRetrievalError) as nse:
nm.retrieve(Query("test"))
assert "Expected" == str(nse.value)
def test_store_retrieve_unexpected_exception_handling(self):
nm = NotesManager(NotesStorageErrorImpl(), get_token)
with pytest.raises(NotesStorageError) as nse:
n = Note(
assoc_id="test1/11111/aaa",
subject="store_retrieve_1",
sub_type="test",
note_val="this is my note 1"
)
n.note_id = None
nm.store(n)
assert "Unhandled" in str(nse.value)
with pytest.raises(NotesRetrievalError) as nse:
nm.retrieve(Query("test"))
assert "Unhandled" in str(nse.value)
def test_store_retrieve_basic(self):
nm = NotesManager(MemoryNotesStorage(), get_token)
nm.store(Note(
assoc_id="test1/11111/aaa",
subject="store_retrieve_1",
sub_type="test",
note_val="this is my note 1"
))
n_list = nm.retrieve(Query("test1"))
assert len(n_list) == 1
assert n_list[0].subject == "store_retrieve_1"
n_list = nm.retrieve(Query("test2"))
assert len(n_list) == 0
def test_store_retrieve_basic_verbosity(self):
nm = NotesManager(MemoryNotesStorage(), get_token)
nm.store(Note(
assoc_id="test1/11111/aaa",
subject="store_retrieve_1",
sub_type="test",
note_val="this is my note 1",
verbosity=5
))
n_list = nm.retrieve(Query("test1", max_verbosity=3))
assert len(n_list) == 0
def test_store_bad_verbosity(self):
nm = NotesManager(MemoryNotesStorage(), get_token)
with pytest.raises(NotesStorageError) as nse:
nm.store(Note(
assoc_id="test1/11111/aaa",
subject="store_retrieve_1",
sub_type="test",
note_val="this is my note 1",
verbosity=-4
))
assert "Verbosity of notes must" in str(nse.value)
with pytest.raises(NotesStorageError) as nse:
nm.store(Note(
assoc_id="test1/11111/aaa",
subject="store_retrieve_1",
sub_type="test",
note_val="this is my note 1",
verbosity=6
))
assert "Verbosity of notes must" in str(nse.value)
def test_store_retrieve_multi(self):
nm = NotesManager(MemoryNotesStorage(), get_token)
nm.store(Note(
assoc_id="test1/11111/aaa",
subject="store_retrieve",
sub_type="test",
note_val="this is my note 1"
))
nm.store(Note(
assoc_id="test1/11111/bbb",
subject="store_retrieve",
sub_type="test",
note_val="this is my note 2"
))
nm.store(Note(
assoc_id="test2/2222/aaa",
subject="store_retrieve_2",
sub_type="test",
note_val="this is my note 3"
))
n_list = nm.retrieve(Query("test2"))
assert len(n_list) == 1
assert n_list[0].subject == "store_retrieve_2"
n_list = nm.retrieve(Query("test1"))
assert len(n_list) == 2
n_list = nm.retrieve(Query("test1", exact_match=True))
assert len(n_list) == 0
n_list = nm.retrieve(Query("test1/11111/aaa", exact_match=True))
assert len(n_list) == 1
@responses.activate
def test_store_retrieve_urls(self):
responses.add(
method="GET",
url="http://test.test",
body="Hello from testland",
status=200,
content_type="text/plain"
)
responses.add(
method="GET",
url="http://test.test2",
body="Hello from testland2",
status=200,
content_type="text/plain"
)
nm = NotesManager(MemoryNotesStorage(), get_token)
nm.store(Note(
assoc_id="test1/11111/aaa",
subject="store_retrieve3",
sub_type="test",
note_val="this is my note 1",
link_url="http://test.test/"
))
nm.store(Note(
assoc_id="test1/11111/bbb",
subject="store_retrieve3",
sub_type="test",
note_val="this is my note 2",
link_url="http://test.test2/"
))
n_list = nm.retrieve(Query("test1"))
assert len(n_list) == 2
for n in n_list:
assert n.resolved_url_value.startswith("Hello from testland")
with pytest.raises(KeyError):
auth_hdr = responses.calls[0].request.headers['X-Auth-Token']
@responses.activate
def test_store_retrieve_url_bad_status_code(self):
responses.add(
method="GET",
url="http://test.test",
body="What note?",
status=404,
content_type="text/plain"
)
responses.add(
method="GET",
url="http://test.test2",
body="Hello from testland2",
status=200,
content_type="text/plain"
)
nm = NotesManager(MemoryNotesStorage(), get_token)
nm.store(Note(
assoc_id="test1/11111/aaa",
subject="store_retrieve3",
sub_type="test",
note_val="this is my note 1",
link_url="http://test.test/"
))
nm.store(Note(
assoc_id="test1/11111/bbb",
subject="store_retrieve3",
sub_type="test",
note_val="this is my note 2",
link_url="http://test.test2/"
))
n_list = nm.retrieve(Query("test1"))
assert len(n_list) == 2
for n in n_list:
if n.assoc_id == "test1/11111/aaa":
assert "failed with status code: 404" in n.resolved_url_value
else:
assert n.resolved_url_value.startswith("Hello from testland")
@responses.activate
def test_store_retrieve_url_does_not_exist(self):
responses.add(
method="GET",
url="http://test.test2",
body="Hello from testland2",
status=200,
content_type="text/plain"
)
nm = NotesManager(MemoryNotesStorage(), get_token)
nm.store(Note(
assoc_id="test1/11111/aaa",
subject="store_retrieve3",
sub_type="test",
note_val="this is my note 1",
link_url="test_breakage://test.test/"
))
nm.store(Note(
assoc_id="test1/11111/bbb",
subject="store_retrieve3",
sub_type="test",
note_val="this is my note 2",
link_url="http://test.test2/"
))
n_list = nm.retrieve(Query("test1"))
assert len(n_list) == 2
for n in n_list:
if n.assoc_id == "test1/11111/aaa":
assert "URL lookup was unable" in n.resolved_url_value
else:
assert n.resolved_url_value.startswith("Hello from testland")
@responses.activate
def test_store_retrieve_with_auth(self):
responses.add(
method="GET",
url="http://test.test2",
body="Hello from testland2",
status=200,
content_type="text/plain"
)
nm = NotesManager(MemoryNotesStorage(), get_token)
nm.store(Note(
assoc_id="test1/11111/bbb",
subject="store_retrieve3",
sub_type="test",
note_val="this is my note 2",
link_url="http://test.test2/",
is_auth_link=True
))
n_list = nm.retrieve(Query("test1"))
assert len(n_list) == 1
for n in n_list:
assert n.resolved_url_value == "Hello from testland2"
auth_hdr = responses.calls[0].request.headers['X-Auth-Token']
assert 'token' == auth_hdr
def test_note_view(self):
nm = NotesManager(MemoryNotesStorage(), get_token)
nm.store(Note(
assoc_id="test1/11111/aaa",
subject="store_retrieve_1",
sub_type="test",
note_val="this is my note 1"
))
n_list = nm.retrieve(Query("test1"))
assert len(n_list) == 1
nid = n_list[0].note_id
nt = n_list[0].note_timestamp
assert n_list[0].view() == {
'assoc_id': 'test1/11111/aaa',
'subject': 'store_retrieve_1',
'sub_type': 'test',
'note_val': 'this is my note 1',
'verbosity': 1,
'note_id': nid,
'note_timestamp': nt,
'resolved_url_value': None,
}

1
src/bin/shipyard_airflow/tests/unit/control/test_action_validators.py

@ -59,7 +59,6 @@ def get_doc_returner(style, ds_name):
# if passed a name of 'defaulted' clear the section
if ds_name == 'defaulted':
dc.data["physical_provisioner"] = None
print(dc.__dict__)
return [dc]
elif doc == 'dep-strat':
return [strategy]

41
src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py

@ -24,6 +24,11 @@ from oslo_config import cfg
import pytest
import responses
from shipyard_airflow.common.notes.notes import NotesManager
from shipyard_airflow.common.notes.notes_helper import NotesHelper
from shipyard_airflow.common.notes.storage_impl_mem import (
MemoryNotesStorage
)
from shipyard_airflow.control.action import actions_api
from shipyard_airflow.control.action.actions_api import ActionsResource
from shipyard_airflow.control.base import ShipyardRequestContext
@ -43,6 +48,15 @@ CONF = cfg.CONF
LOG = logging.getLogger(__name__)
def get_token():
"""Stub method to use for NotesHelper/NotesManager"""
return "token"
# Notes helper that can be mocked into various objects to prevent database
# dependencies
nh = NotesHelper(NotesManager(MemoryNotesStorage(), get_token))
def create_req(ctx, body):
'''creates a falcon request'''
env = testing.create_environ(
@ -283,8 +297,6 @@ def test_get_all_actions():
action_resource.get_all_actions_db = actions_db
action_resource.get_all_dag_runs_db = dag_runs_db
action_resource.get_all_tasks_db = tasks_db
os.environ['DB_CONN_AIRFLOW'] = 'nothing'
os.environ['DB_CONN_SHIPYARD'] = 'nothing'
result = action_resource.get_all_actions()
assert len(result) == len(actions_db())
for action in result:
@ -296,6 +308,31 @@ def test_get_all_actions():
assert action['dag_status'] == 'SUCCESS'
@mock.patch('shipyard_airflow.control.action.actions_api.notes_helper',
new=nh)
def test_get_all_actions_notes(*args):
"""
Tests the main response from get all actions
"""
action_resource = ActionsResource()
action_resource.get_all_actions_db = actions_db
action_resource.get_all_dag_runs_db = dag_runs_db
action_resource.get_all_tasks_db = tasks_db
# inject some notes
nh.make_action_note('aaaaaa', "hello from aaaaaa1")
nh.make_action_note('aaaaaa', "hello from aaaaaa2")
nh.make_action_note('bbbbbb', "hello from bbbbbb")
result = action_resource.get_all_actions()
assert len(result) == len(actions_db())
for action in result:
if action['id'] == 'aaaaaa':
assert len(action['notes']) == 2
if action['id'] == 'bbbbbb':
assert len(action['notes']) == 1
assert action['notes'][0]['note_val'] == 'hello from bbbbbb'
def _gen_action_resource_stubbed():
# TODO(bryan-strassner): mabye subclass this instead?
action_resource = ActionsResource()

Loading…
Cancel
Save