From 35d6089d325748b88cecabe7150e05e312ffe315 Mon Sep 17 00:00:00 2001 From: Bryan Strassner Date: Mon, 1 Oct 2018 17:46:57 -0500 Subject: [PATCH] Add notes common code for Shipyard Adds the common/shared code to support notes in Shipyard such that this component can be reused between the Shipyard API and the workflows. Includes a single example of creating and retreiving notes as part of action creation, that will likely be changed in a future change. Note that this change doesn't include changes to the CLI to represent notes outwardly, but the first example can be seen using the --output-format=format option (shipyard get actions) Change-Id: I2f87713eb74dae312912ff4c36e6ae30a569ea38 --- doc/source/_static/shipyard.conf.sample | 7 + .../7486ddec1979_create_notes_table.py | 47 +++ .../etc/shipyard/shipyard.conf.sample | 7 + .../shipyard_airflow/common/notes/__init__.py | 0 .../shipyard_airflow/common/notes/errors.py | 44 +++ .../shipyard_airflow/common/notes/notes.py | 318 ++++++++++++++++ .../common/notes/notes_helper.py | 159 ++++++++ .../common/notes/storage_impl_db.py | 167 +++++++++ .../common/notes/storage_impl_mem.py | 43 +++ .../shipyard_airflow/conf/config.py | 11 + .../control/action/actions_api.py | 9 + .../shipyard_airflow/control/helpers/notes.py | 49 +++ .../tests/unit/common/notes/__init__.py | 0 .../tests/unit/common/notes/test_notes.py | 353 ++++++++++++++++++ .../unit/control/test_action_validators.py | 1 - .../tests/unit/control/test_actions_api.py | 41 +- 16 files changed, 1253 insertions(+), 3 deletions(-) create mode 100644 src/bin/shipyard_airflow/alembic/versions/7486ddec1979_create_notes_table.py create mode 100644 src/bin/shipyard_airflow/shipyard_airflow/common/notes/__init__.py create mode 100644 src/bin/shipyard_airflow/shipyard_airflow/common/notes/errors.py create mode 100644 src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes.py create mode 100644 src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes_helper.py create mode 100644 src/bin/shipyard_airflow/shipyard_airflow/common/notes/storage_impl_db.py create mode 100644 src/bin/shipyard_airflow/shipyard_airflow/common/notes/storage_impl_mem.py create mode 100644 src/bin/shipyard_airflow/shipyard_airflow/control/helpers/notes.py create mode 100644 src/bin/shipyard_airflow/tests/unit/common/notes/__init__.py create mode 100644 src/bin/shipyard_airflow/tests/unit/common/notes/test_notes.py diff --git a/doc/source/_static/shipyard.conf.sample b/doc/source/_static/shipyard.conf.sample index eac4bfc3..f094fad5 100644 --- a/doc/source/_static/shipyard.conf.sample +++ b/doc/source/_static/shipyard.conf.sample @@ -369,6 +369,13 @@ # Airship component validation timeout (in seconds) (integer value) #validation_read_timeout = 300 +# Maximum time to wait to connect to a note source URL (in seconds) (integer +# value) +#notes_connect_timeout = 5 + +# Read timeout for a note source URL (in seconds) (integer value) +#notes_read_timeout = 10 + [shipyard] diff --git a/src/bin/shipyard_airflow/alembic/versions/7486ddec1979_create_notes_table.py b/src/bin/shipyard_airflow/alembic/versions/7486ddec1979_create_notes_table.py new file mode 100644 index 00000000..ba83c537 --- /dev/null +++ b/src/bin/shipyard_airflow/alembic/versions/7486ddec1979_create_notes_table.py @@ -0,0 +1,47 @@ +"""create notes table + +Revision ID: 7486ddec1979 +Revises: 51b92375e5c4 +Create Date: 2018-10-02 16:10:08.139378 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy import (types, func) + + +# revision identifiers, used by Alembic. +revision = '7486ddec1979' +down_revision = '51b92375e5c4' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'notes', + # ULID key for the note + sa.Column('note_id', types.String(26), primary_key=True), + # The supplied association id used for lookup + sa.Column('assoc_id', types.String(128), nullable=False), + # The supplied subject of the note (what is this note about?) + sa.Column('subject', types.String(128), nullable=False), + # The supplied type of the subject (what kind of thing is the subject?) + sa.Column('sub_type', types.String(128), nullable=False), + # The text value of the note + sa.Column('note_val', sa.Text, nullable=False), + # The numeric verbosity level of the note (1-5) + sa.Column('verbosity', types.Integer, nullable=False), + # An optional URL containing more info for the note + sa.Column('link_url', sa.Text, nullable=True), + # Boolean if the link requires a X-Auth-Token header + sa.Column('is_auth_link', types.Boolean, nullable=False), + # The creation timestamp for the note + sa.Column('note_timestamp', + types.TIMESTAMP(timezone=True), + server_default=func.now()), + ) + + +def downgrade(): + op.drop_table('notes') diff --git a/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample b/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample index eac4bfc3..f094fad5 100644 --- a/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample +++ b/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample @@ -369,6 +369,13 @@ # Airship component validation timeout (in seconds) (integer value) #validation_read_timeout = 300 +# Maximum time to wait to connect to a note source URL (in seconds) (integer +# value) +#notes_connect_timeout = 5 + +# Read timeout for a note source URL (in seconds) (integer value) +#notes_read_timeout = 10 + [shipyard] diff --git a/src/bin/shipyard_airflow/shipyard_airflow/common/notes/__init__.py b/src/bin/shipyard_airflow/shipyard_airflow/common/notes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/bin/shipyard_airflow/shipyard_airflow/common/notes/errors.py b/src/bin/shipyard_airflow/shipyard_airflow/common/notes/errors.py new file mode 100644 index 00000000..d79f2008 --- /dev/null +++ b/src/bin/shipyard_airflow/shipyard_airflow/common/notes/errors.py @@ -0,0 +1,44 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Errors for the Notes component""" + + +class NotesError(Exception): + """Base exception for all NotesErrors""" + pass + + +class NotesInitializationError(NotesError): + """NotesInitializationError + + Raised for errors while initializing a Notes instance + """ + pass + + +class NotesRetrievalError(NotesError): + """NotesRetrievalError + + Raised when there is an error retrieving notes + """ + pass + + +class NotesStorageError(NotesError): + """NotesStorageError + + Raised when there is an error attempting to store a note. + """ + pass diff --git a/src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes.py b/src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes.py new file mode 100644 index 00000000..af057d8f --- /dev/null +++ b/src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes.py @@ -0,0 +1,318 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Notes + +A reusable component allowing for recording and retreiving information related +to arbitrarily useage-based keys (the usage makes the association keys to their +string-based liking). The intention for Notes is generally for additional +non-fielded information that may be of interest to a user. Notes are not +intended to store info that would drive code paths or decisions. This is not +an arbitrary use key-value database. +""" +import abc +from datetime import datetime +import logging + +import requests +from requests.exceptions import HTTPError +from requests.exceptions import RequestException +import ulid + +from .errors import NotesInitializationError +from .errors import NotesRetrievalError +from .errors import NotesStorageError + +LOG = logging.getLogger(__name__) +MAX_VERBOSITY = 5 +MIN_VERBOSITY = 1 + + +class NotesManager: + """Interface to store and retrieve notes + + :param storage: A NotesStorage object to store and retrieve notes for a + specific storage mechanism. e.g. Database, Service + :param get_token: A method that returns an auth token that will be used + as the X-Auth-Token header when resolving url-based notes + :param connect_timeout: optional, The maximum time waiting to connect to + a URL. Defaults to 3 seconds + :param read_timeout: optional, The maximum time waiting to read the info + from a URL. Defaults to 10 seconds + + Example usage: + nm = NotesManager(SQLNotesStorage("connection_info"), get_url) + a_note = nm.store(Note(...params...)) + notes = list(nm.retrieve(Query("some/id"))) + """ + def __init__(self, storage, get_token, connect_timeout=None, + read_timeout=None): + if not isinstance(storage, NotesStorage): + raise NotesInitializationError( + "Storage object is not suitable for use with Notes" + ) + self.storage = storage + LOG.info( + "Initializing Notes with storage mechanism: %s", + storage.__class__.__name__ + ) + + if not callable(get_token): + raise NotesInitializationError( + "Parameter get_token is not suitable for use with Notes. " + "Must be a callable." + ) + self.get_token = get_token + + # connect and read timeouts default to 3 and 10 seconds + self.connect_timeout = connect_timeout or 3 + self.read_timeout = read_timeout or 10 + + def create(self, assoc_id, subject, sub_type, note_val, + verbosity=None, link_url=None, is_auth_link=None, + note_id=None, note_timestamp=None, store=True): + """Creates and stores a Note object from parameters + + Passthrough helper method to avoid additional imports for the Note + class. Most of the parameters match that of the Note constructor. + See: func:`notes.Note.__init__` + Additional Parameters: + + :param store: optinal, default=True, invoke the store method + immediately upon creation, if true + """ + n = Note(assoc_id, subject, sub_type, note_val, + verbosity=None, link_url=None, is_auth_link=None, + note_id=None, note_timestamp=None) + if store: + return self.store(n) + else: + return n + + def store(self, note): + """Store a note + + :param note: A Note object to store + :returns: The note, as it was after storage + """ + if note.verbosity < MIN_VERBOSITY or note.verbosity > MAX_VERBOSITY: + raise NotesStorageError( + "Verbosity of notes must range from {} " + "to {} (most verbose)".format(MIN_VERBOSITY, MAX_VERBOSITY)) + try: + return self.storage.store(note) + except NotesStorageError: + raise + except Exception as ex: + LOG.exception(ex) + raise NotesStorageError("Unhandled error during storage of a note") + + def retrieve(self, query): + """Retrieve a list of notes + + :param query: a query object to retrieve notes + """ + try: + notes = list(self.storage.retrieve(query)) + except NotesRetrievalError: + raise + except Exception as ex: + LOG.exception(ex) + raise NotesRetrievalError( + "Unhandled error during retrieval of notes" + ) + # Get the auth token once per retrieve, not once per note. + if notes: + auth_token = self.get_token() + # resolve the note urls + # TODO: threaded? + for note in notes: + self._resolve_note_url(note, auth_token) + return notes + + def _resolve_note_url(self, note, auth_token): + """Resolve and set the value obtained from the URL for a Note. + + :param note: the Note object to retreive and set the value for. + :param auth_token: the authorization token set as a header for the URL + request if one is indicated as needed by the note. + + If there is data retrieved at the note's url, set the + resolved_url_value with those contents. + + If there is no url for the note, return, with resolved_url_value as + None + + If there is no data retrieved, resolved_url_value for the note remains + None + + If there is an error related to retreiving the note's url value, the + resolved_url_value is set to a placeholder value indicating that the + value could not be obtained. + """ + if not isinstance(note, Note): + LOG.debug( + "Note is None or not a Note object. URL will not be resolved" + ) + return + + if not note.link_url: + LOG.debug("Note %s has no link to resolve", note.note_id) + return + + contents = None + try: + headers = {} + + # Don't pass credentials if not needed. + if note.is_auth_link: + headers['X-Auth-Token'] = auth_token + + response = requests.get( + note.link_url, + headers=headers, + timeout=(self.connect_timeout, self.read_timeout)) + response.raise_for_status() + + # Set the valid response text to the note + note.resolved_url_value = response.text + + except HTTPError as he: + # A bad status code - don't stop, but log and indicate in note. + LOG.info( + "Note %s has a url returning a bad status code: %s", + note.note_id, response.status_code + ) + note.resolved_url_value = ( + "Note contents could not be retrieved. URL lookup failed " + "with status code: {}" + ).format(response.status_code) + except RequestException as rex: + # A more serious exception; log and indicate in the note + LOG.exception(rex) + note.resolved_url_value = ( + "Note contents could not be retrieved. URL lookup was unable " + "to complete" + ) + except Exception as ex: + # Whatever's left, log and indicate in the note + LOG.exception(ex) + note.resolved_url_value = ( + "Note contents could not be retrieved due to unexpected " + "circumstances" + ) + + +class Note: + """Model object representing a note + + :param assoc_id: arbitrary value like action/xxxxxxx or + step/xxxxxxx/step_name, useful for lookup, set by note creator + Limit: 128 characters + :param subject: arbitrary value to be used as the subject of the note, + useful to a human e.g. mtn15r11n0001, set by note creator + Limit: 128 characters + :param sub_type: arbitrary value used to qualify the subject of the note, + e.g. Node, Action, Step, set by note creator + Limit: 128 characters + :param note_val: the text value of the note, the contents of info to be + displayed as note + :param link_url: optional url that should be followed when the note is + retrieved to append to its value + :param is_auth_link: boolean if Shipyard's service ID auth credentials are + needed to make the call to follow the link default=false + :param verbosity: integer, 1-5 indicating the verbosity level, default = 1 + :param note_id: ULID that uniquely represents a note. Users are not + expected to pass a value for the ID of a note, it will be assigned + :param note_timestamp: String representation of the timestamp for the note + """ + def __init__(self, assoc_id, subject, sub_type, note_val, + verbosity=None, link_url=None, is_auth_link=None, + note_id=None, note_timestamp=None): + self.assoc_id = assoc_id + self.subject = subject + self.sub_type = sub_type + self.note_val = note_val + self.verbosity = verbosity or MIN_VERBOSITY + self.link_url = link_url + self.is_auth_link = is_auth_link or False + self.note_id = note_id or ulid.ulid() + self.note_timestamp = note_timestamp or str(datetime.utcnow()) + self._resolved_url_value = None + + @property + def resolved_url_value(self): + return self._resolved_url_value + + @resolved_url_value.setter + def resolved_url_value(self, value): + self._resolved_url_value = value + + def view(self): + """Returns the user-facing dictionary version of the Note""" + return { + 'assoc_id': self.assoc_id, + 'subject': self.subject, + 'sub_type': self.sub_type, + 'note_val': self.note_val, + 'verbosity': self.verbosity, + 'note_id': self.note_id, + 'note_timestamp': self.note_timestamp, + 'resolved_url_value': self.resolved_url_value, + } + + +class Query: + """Model object for a query to retrieve notes + + :param assoc_id_pattern: The pattern to match to the assoc_id for a note. + :param max_verbosity: optional integer 1-5, defaults to 5 (everything) + :param exact_match: boolean, defaults to False. If true, the + assoc_id_pattern will be used precisely, otherwise assoc_id_pattern + will be matched to the start of the assoc_id for notes. + """ + def __init__(self, assoc_id_pattern, max_verbosity=None, exact_match=None): + self.assoc_id_pattern = assoc_id_pattern + self.max_verbosity = max_verbosity or MAX_VERBOSITY + self.exact_match = exact_match or False + + +class NotesStorage(metaclass=abc.ABCMeta): + """NotesStorage abstract base class + + Defines the interface for NotesStorage implementations that provide the + specific mappings of Note objects to the target data store. + """ + @abc.abstractmethod + def store(self, note): + """Store a Note object, return the note object as stored + + :param note: a Note object + :returns: A single Note object, as was persisted. + :raises NotesStorageError: When there is a failure to create the note. + """ + pass + + @abc.abstractmethod + def retrieve(self, query): + """Query for a list of Note objects + + :param query: a Notes Query object representing the notes to be + retrieved. + :returns: List of Note objects matching the query + :raises NotesRetrievalError: when there is a failure to retrieve notes, + however an empty list is expected to be returned in the case of no + results. + """ + pass diff --git a/src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes_helper.py b/src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes_helper.py new file mode 100644 index 00000000..bbdb7ec0 --- /dev/null +++ b/src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes_helper.py @@ -0,0 +1,159 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import logging + +from .notes import MAX_VERBOSITY +from .notes import MIN_VERBOSITY +from .notes import Query + +LOG = logging.getLogger(__name__) + +# Constants and magic numbers for actions: +# [7:33] to slice a string like: +# action/12345678901234567890123456 +# matching the patterns in this helper. +ACTION_KEY_PATTERN = "action/{}" +ACTION_LOOKUP_PATTERN = "action/" +ACTION_ID_START = 7 +ACTION_ID_END = 33 + + +class NotesHelper: + """Notes Helper + + Provides helper methods for the common use cases for notes + :param notes_manager: the NotesManager object to use + """ + def __init__(self, notes_manager): + self.nm = notes_manager + + def _failsafe_make_note(self, assoc_id, subject, sub_type, note_val, + verbosity=None, link_url=None, is_auth_link=None): + """LOG and continue on any note creation failure""" + try: + self.nm.create( + assoc_id=assoc_id, + subject=subject, + sub_type=sub_type, + note_val=note_val, + verbosity=verbosity, + link_url=link_url, + is_auth_link=is_auth_link, + ) + except Exception as ex: + LOG.warn( + "Creating note for {} encountered a problem, exception info " + "follows, but processing is not halted for notes.", + assoc_id + ) + LOG.exception(ex) + + def _failsafe_get_notes(self, assoc_id_pattern, max_verbosity, + exact_match): + """LOG and continue on any note retrieval failure""" + try: + q = Query(assoc_id_pattern, max_verbosity, exact_match) + return self.nm.retrieve(q) + except Exception as ex: + LOG.warn( + "Note retrieval for {} encountered a problem, exception " + "info follows, but processing is not halted for notes.", + assoc_id_pattern + ) + LOG.exception(ex) + return [] + + def make_action_note(self, action_id, note_val, subject=None, + sub_type=None, verbosity=None, link_url=None, + is_auth_link=None): + """Creates an action note using a convention for the note's assoc_id + + :param action_id: the ULID id of an action + :param note_val: the text for the note + :param subject: optional subject for the note. Defaults to the + action_id + :param sub_type: optional subject type for the note, defaults to + "action metadata" + :param verbosity: optional verbosity for the note, defaults to 1, + i.e.: summary level + :param link_url: optional link URL if there's additional information + to retreive from another source. + :param is_auth_link: optional, defaults to False, indicating if there + is a need to send a Shipyard service account token with the + request to the optional URL + """ + assoc_id = ACTION_KEY_PATTERN.format(action_id) + if subject is None: + subject = action_id + if sub_type is None: + sub_type = "action metadata" + if verbosity is None: + verbosity = 1 + + self._failsafe_make_note( + assoc_id=assoc_id, + subject=subject, + sub_type=sub_type, + note_val=note_val, + verbosity=verbosity, + link_url=link_url, + is_auth_link=is_auth_link, + ) + + def get_all_action_notes(self, verbosity=None): + """Retrieve notes for all actions, in a dictionary keyed by action id. + + :param verbosity: optional, 1-5, the maximum verbosity level to + retrieve, defaults to 1 (most summary level) + + Warning: if there are a lot of URL links in notes, this could take a + long time. The default verbosity of 1 attempts to avoid this as there + is less expectation of URL links on summary notes. + """ + max_verbosity = verbosity or MIN_VERBOSITY + notes = self._failsafe_get_notes( + assoc_id_pattern=ACTION_LOOKUP_PATTERN, + max_verbosity=verbosity, + exact_match=False + ) + note_dict = {} + for n in notes: + # magic numbers [7:33] to slice a string like: + # action/12345678901234567890123456/something + # matching the convention in this helper. + # in the case where there are non-compliant, the slice will make + # the action_id a garbage key and that note will not be easily + # associated. + action_id = n.assoc_id[ACTION_ID_START:ACTION_ID_END] + if action_id not in note_dict: + note_dict[action_id] = [] + note_dict[action_id].append(n) + return note_dict + + def get_action_notes(self, action_id, verbosity=None): + """Retrive notes related to a particular action + + :param action_id: the action for which to retrieve notes. + :param verbosity: optional, 1-5, the maximum verbosity level to + retrieve, defaults to 5 (most detailed level) + """ + assoc_id_pattern = ACTION_KEY_PATTERN.format(action_id) + max_verbosity = verbosity or MAX_VERBOSITY + exact_match = True + return self._failsafe_get_notes( + assoc_id_pattern=assoc_id_pattern, + max_verbosity=max_verbosity, + exact_match=exact_match + ) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/common/notes/storage_impl_db.py b/src/bin/shipyard_airflow/shipyard_airflow/common/notes/storage_impl_db.py new file mode 100644 index 00000000..c7577317 --- /dev/null +++ b/src/bin/shipyard_airflow/shipyard_airflow/common/notes/storage_impl_db.py @@ -0,0 +1,167 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""ShipyardSQLNotesStorage + +Implementation of NotesStorage that is based on the structure of the notes +table as defined in Shipyard. Accepts a SQLAlchemy engine as input, and +generates a model class from the notes table. + +Mapping to/from Note objects is encapsulated here for a consistent interface +""" +from contextlib import contextmanager +import logging + +from sqlalchemy import and_ +from sqlalchemy import Column +from sqlalchemy import func +from sqlalchemy import Text +from sqlalchemy import types + +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker + +from .notes import Note +from .notes import NotesStorage +from .errors import NotesError +from .errors import NotesInitializationError + +LOG = logging.getLogger(__name__) +Base = declarative_base() + + +class TNote(Base): + """Notes ORM class""" + __tablename__ = 'notes' + + # These must align with the table defined using Alembic + note_id = Column('note_id', types.String(26), primary_key=True) + assoc_id = Column('assoc_id', types.String(128), nullable=False) + subject = Column('subject', types.String(128), nullable=False) + sub_type = Column('sub_type', types.String(128), nullable=False) + note_val = Column('note_val', Text, nullable=False) + verbosity = Column('verbosity', types.Integer, nullable=False) + link_url = Column('link_url', Text, nullable=True) + is_auth_link = Column('is_auth_link', types.Boolean, nullable=False) + note_timestamp = Column('note_timestamp', + types.TIMESTAMP(timezone=True), + server_default=func.now()) + + +class ShipyardSQLNotesStorage(NotesStorage): + """SQL Alchemy implementation of a notes storage; Shipayrd table structure + + Accepts a SQL Alchemy Engine to serve as the connection to the persistence + layer for Notes. + + :param engine_getter: A method that can be used to get SQLAlchemy engine + to use + """ + def __init__(self, engine_getter): + try: + self._engine_getter = engine_getter + self._session = None + except Exception as ex: + LOG.exception(ex) + raise NotesInitializationError( + "Misconfiguration has casuse a failure to setup the desired " + "database connection for Notes." + ) + + def _get_session(self): + """Lazy initilize the sessionmaker, invoke the engine getter, and + use it to return a session + """ + if not self._session: + self._session = sessionmaker(bind=self._engine_getter()) + return self._session() + + @contextmanager + def session_scope(self): + """Context manager for a SQLAlchemy session""" + session = self._get_session() + try: + yield session + session.commit() + except Exception as ex: + session.rollback() + if isinstance(ex, NotesError): + raise + else: + LOG.exception(ex) + raise NotesError( + "An unexpected error has occurred while attempting to " + "interact with the database for note storage" + ) + finally: + session.close() + + def store(self, note): + """Store a note in the database""" + r_note = None + with self.session_scope() as session: + tnote = self._map(note, TNote) + session.add(tnote) + r_note = self._map(tnote, Note) + return r_note + + def retrieve(self, query): + a_id_pat = query.assoc_id_pattern + m_verb = query.max_verbosity + r_notes = [] + with self.session_scope() as session: + notes_res = [] + if (query.exact_match): + n_qry = session.query(TNote).filter( + and_( + TNote.assoc_id == a_id_pat, + TNote.verbosity <= m_verb + ) + ) + else: + n_qry = session.query(TNote).filter( + and_( + TNote.assoc_id.like(a_id_pat + '%'), + TNote.verbosity <= m_verb + ) + ) + db_notes = n_qry.all() + for tn in db_notes: + r_notes.append(self._map(tn, Note)) + return r_notes + + def _map(self, src, target_type): + """Maps a Note object to/from a TNote object. + + :param src: the object to use as a source + :param target_type: the type of object to create and map to + """ + try: + tgt = target_type( + assoc_id=src.assoc_id, + subject=src.subject, + sub_type=src.sub_type, + note_val=src.note_val, + verbosity=src.verbosity, + link_url=src.link_url, + is_auth_link=src.is_auth_link, + note_id=src.note_id, + note_timestamp=src.note_timestamp + ) + except AttributeError as ae: + LOG.exception(ae) + raise NotesError( + "Note could not be translated from/to SQL form; mapping error" + ) + return tgt diff --git a/src/bin/shipyard_airflow/shipyard_airflow/common/notes/storage_impl_mem.py b/src/bin/shipyard_airflow/shipyard_airflow/common/notes/storage_impl_mem.py new file mode 100644 index 00000000..947bd0e4 --- /dev/null +++ b/src/bin/shipyard_airflow/shipyard_airflow/common/notes/storage_impl_mem.py @@ -0,0 +1,43 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Implementations of the NotesStorage base class""" +from .notes import NotesStorage + + +class MemoryNotesStorage(NotesStorage): + """In-memory note storage + + Primarily useful for testing + """ + def __init__(self): + self.storage = {} + + def store(self, note): + self.storage[note.note_id] = note + return note + + def retrieve(self, query): + pat = query.assoc_id_pattern + mv = query.max_verbosity + notes = [] + if query.exact_match: + for note in self.storage.values(): + if note.assoc_id == pat and note.verbosity <= mv: + notes.append(note) + else: + for note in self.storage.values(): + if note.assoc_id.startswith(pat) and note.verbosity <= mv: + notes.append(note) + return notes diff --git a/src/bin/shipyard_airflow/shipyard_airflow/conf/config.py b/src/bin/shipyard_airflow/shipyard_airflow/conf/config.py index 33092545..cc641786 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/conf/config.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/conf/config.py @@ -228,6 +228,17 @@ SECTIONS = [ default=300, help='Airship component validation timeout (in seconds)' ), + cfg.IntOpt( + 'notes_connect_timeout', + default=5, + help=('Maximum time to wait to connect to a note source URL ' + '(in seconds)') + ), + cfg.IntOpt( + 'notes_read_timeout', + default=10, + help='Read timeout for a note source URL (in seconds)' + ), ] ), ConfigSection( diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py b/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py index aa23b583..b1b07a12 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py @@ -32,6 +32,7 @@ from shipyard_airflow.control.base import BaseResource from shipyard_airflow.control.helpers import configdocs_helper from shipyard_airflow.control.helpers.configdocs_helper import ( ConfigdocsHelper) +from shipyard_airflow.control.helpers.notes import NOTES as notes_helper from shipyard_airflow.control.json_schemas import ACTION from shipyard_airflow.db.db import AIRFLOW_DB, SHIPYARD_DB from shipyard_airflow.errors import ApiError @@ -182,6 +183,10 @@ class ActionsResource(BaseResource): # insert the action into the shipyard db self.insert_action(action=action) + notes_helper.make_action_note( + action_id=action['id'], + note_val="Configdoc revision {}".format(action['committed_rev_id']) + ) self.audit_control_command_db({ 'id': ulid.ulid(), 'action_id': action['id'], @@ -202,6 +207,7 @@ class ActionsResource(BaseResource): all_dag_runs = self.get_dag_run_map() all_tasks = self.get_all_tasks_db() + notes = notes_helper.get_all_action_notes(verbosity=1) # correlate the actions and dags into a list of action entites actions = [] @@ -222,6 +228,9 @@ class ActionsResource(BaseResource): '%Y-%m-%dT%H:%M:%S') == dag_key_date ] action['steps'] = format_action_steps(action_id, action_tasks) + action['notes'] = [] + for note in notes.get(action_id, []): + action['notes'].append(note.view()) actions.append(action) return actions diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/notes.py b/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/notes.py new file mode 100644 index 00000000..6bfb0092 --- /dev/null +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/notes.py @@ -0,0 +1,49 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Notes + +Provides setup and access to a NotesHelper object that is used across the +API +""" +import logging + +from oslo_config import cfg + +from shipyard_airflow.common.notes.notes import NotesManager +from shipyard_airflow.common.notes.notes_helper import NotesHelper +from shipyard_airflow.common.notes.storage_impl_db import ( + ShipyardSQLNotesStorage +) +from shipyard_airflow.control.service_endpoints import get_token +from shipyard_airflow.db.db import SHIPYARD_DB + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + + +def _notes_manager(): + """Setup a NotesManager object using Shipyard settings""" + sy_engine_getter = SHIPYARD_DB.get_engine + return NotesManager( + ShipyardSQLNotesStorage(sy_engine_getter), + get_token, + CONF.requests_config.notes_connect_timeout, + CONF.requests_config.notes_read_timeout + ) + + +# NOTES is the notes manager that can be imported and used by other modules +# for notes functionality across the Shipyard API. +NOTES = NotesHelper(_notes_manager()) diff --git a/src/bin/shipyard_airflow/tests/unit/common/notes/__init__.py b/src/bin/shipyard_airflow/tests/unit/common/notes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/bin/shipyard_airflow/tests/unit/common/notes/test_notes.py b/src/bin/shipyard_airflow/tests/unit/common/notes/test_notes.py new file mode 100644 index 00000000..397bd117 --- /dev/null +++ b/src/bin/shipyard_airflow/tests/unit/common/notes/test_notes.py @@ -0,0 +1,353 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for the Notes component""" +from unittest import mock + +import pytest +import responses + +from shipyard_airflow.common.notes.errors import ( + NotesInitializationError, + NotesRetrievalError, + NotesStorageError +) +from shipyard_airflow.common.notes.notes import ( + Note, + NotesManager, + NotesStorage, + Query +) +from shipyard_airflow.common.notes.storage_impl_mem import MemoryNotesStorage + + +def get_token(): + return "token" + + +class NotesStorageErrorImpl(NotesStorage): + def store(self, note): + raise Exception("Didn't see it coming") + + def retrieve(self, query): + raise Exception("Outta Nowhere") + + +class NotesStorageExpectedErrorImpl(NotesStorage): + def store(self, note): + raise NotesStorageError("Expected") + + def retrieve(self, query): + raise NotesRetrievalError("Expected") + + +class TestNotesManager: + def test_init(self): + with pytest.raises(NotesInitializationError) as nie: + nm = NotesManager(None, None) + assert "Storage object is not" in str(nie.value) + + with pytest.raises(NotesInitializationError) as nie: + nm = NotesManager({}, None) + assert "Storage object is not" in str(nie.value) + + with pytest.raises(NotesInitializationError) as nie: + nm = NotesManager(MemoryNotesStorage(), None) + assert "Parameter get_token" in str(nie.value) + + with pytest.raises(NotesInitializationError) as nie: + nm = NotesManager(MemoryNotesStorage(), {}) + assert "Parameter get_token" in str(nie.value) + + nm = NotesManager(MemoryNotesStorage(), get_token) + assert nm.connect_timeout == 3 + assert nm.read_timeout == 10 + + nm = NotesManager(MemoryNotesStorage(), get_token, connect_timeout=99, + read_timeout=999) + assert nm.connect_timeout == 99 + assert nm.read_timeout == 999 + assert isinstance(nm.storage, MemoryNotesStorage) + + def test_store_retrieve_expected_exception_handling(self): + nm = NotesManager(NotesStorageExpectedErrorImpl(), get_token) + with pytest.raises(NotesStorageError) as nse: + n = Note( + assoc_id="test1/11111/aaa", + subject="store_retrieve_1", + sub_type="test", + note_val="this is my note 1" + ) + n.note_id = None + nm.store(n) + assert "Expected" == str(nse.value) + + with pytest.raises(NotesRetrievalError) as nse: + nm.retrieve(Query("test")) + assert "Expected" == str(nse.value) + + def test_store_retrieve_unexpected_exception_handling(self): + nm = NotesManager(NotesStorageErrorImpl(), get_token) + with pytest.raises(NotesStorageError) as nse: + n = Note( + assoc_id="test1/11111/aaa", + subject="store_retrieve_1", + sub_type="test", + note_val="this is my note 1" + ) + n.note_id = None + nm.store(n) + assert "Unhandled" in str(nse.value) + + with pytest.raises(NotesRetrievalError) as nse: + nm.retrieve(Query("test")) + assert "Unhandled" in str(nse.value) + + def test_store_retrieve_basic(self): + nm = NotesManager(MemoryNotesStorage(), get_token) + nm.store(Note( + assoc_id="test1/11111/aaa", + subject="store_retrieve_1", + sub_type="test", + note_val="this is my note 1" + )) + n_list = nm.retrieve(Query("test1")) + assert len(n_list) == 1 + assert n_list[0].subject == "store_retrieve_1" + n_list = nm.retrieve(Query("test2")) + assert len(n_list) == 0 + + def test_store_retrieve_basic_verbosity(self): + nm = NotesManager(MemoryNotesStorage(), get_token) + nm.store(Note( + assoc_id="test1/11111/aaa", + subject="store_retrieve_1", + sub_type="test", + note_val="this is my note 1", + verbosity=5 + )) + n_list = nm.retrieve(Query("test1", max_verbosity=3)) + assert len(n_list) == 0 + + def test_store_bad_verbosity(self): + nm = NotesManager(MemoryNotesStorage(), get_token) + with pytest.raises(NotesStorageError) as nse: + nm.store(Note( + assoc_id="test1/11111/aaa", + subject="store_retrieve_1", + sub_type="test", + note_val="this is my note 1", + verbosity=-4 + )) + assert "Verbosity of notes must" in str(nse.value) + + with pytest.raises(NotesStorageError) as nse: + nm.store(Note( + assoc_id="test1/11111/aaa", + subject="store_retrieve_1", + sub_type="test", + note_val="this is my note 1", + verbosity=6 + )) + assert "Verbosity of notes must" in str(nse.value) + + def test_store_retrieve_multi(self): + nm = NotesManager(MemoryNotesStorage(), get_token) + nm.store(Note( + assoc_id="test1/11111/aaa", + subject="store_retrieve", + sub_type="test", + note_val="this is my note 1" + )) + nm.store(Note( + assoc_id="test1/11111/bbb", + subject="store_retrieve", + sub_type="test", + note_val="this is my note 2" + )) + nm.store(Note( + assoc_id="test2/2222/aaa", + subject="store_retrieve_2", + sub_type="test", + note_val="this is my note 3" + )) + n_list = nm.retrieve(Query("test2")) + assert len(n_list) == 1 + assert n_list[0].subject == "store_retrieve_2" + n_list = nm.retrieve(Query("test1")) + assert len(n_list) == 2 + n_list = nm.retrieve(Query("test1", exact_match=True)) + assert len(n_list) == 0 + n_list = nm.retrieve(Query("test1/11111/aaa", exact_match=True)) + assert len(n_list) == 1 + + @responses.activate + def test_store_retrieve_urls(self): + responses.add( + method="GET", + url="http://test.test", + body="Hello from testland", + status=200, + content_type="text/plain" + ) + responses.add( + method="GET", + url="http://test.test2", + body="Hello from testland2", + status=200, + content_type="text/plain" + ) + + nm = NotesManager(MemoryNotesStorage(), get_token) + nm.store(Note( + assoc_id="test1/11111/aaa", + subject="store_retrieve3", + sub_type="test", + note_val="this is my note 1", + link_url="http://test.test/" + )) + nm.store(Note( + assoc_id="test1/11111/bbb", + subject="store_retrieve3", + sub_type="test", + note_val="this is my note 2", + link_url="http://test.test2/" + )) + n_list = nm.retrieve(Query("test1")) + assert len(n_list) == 2 + for n in n_list: + assert n.resolved_url_value.startswith("Hello from testland") + with pytest.raises(KeyError): + auth_hdr = responses.calls[0].request.headers['X-Auth-Token'] + + @responses.activate + def test_store_retrieve_url_bad_status_code(self): + responses.add( + method="GET", + url="http://test.test", + body="What note?", + status=404, + content_type="text/plain" + ) + responses.add( + method="GET", + url="http://test.test2", + body="Hello from testland2", + status=200, + content_type="text/plain" + ) + + nm = NotesManager(MemoryNotesStorage(), get_token) + nm.store(Note( + assoc_id="test1/11111/aaa", + subject="store_retrieve3", + sub_type="test", + note_val="this is my note 1", + link_url="http://test.test/" + )) + nm.store(Note( + assoc_id="test1/11111/bbb", + subject="store_retrieve3", + sub_type="test", + note_val="this is my note 2", + link_url="http://test.test2/" + )) + n_list = nm.retrieve(Query("test1")) + assert len(n_list) == 2 + for n in n_list: + if n.assoc_id == "test1/11111/aaa": + assert "failed with status code: 404" in n.resolved_url_value + else: + assert n.resolved_url_value.startswith("Hello from testland") + + @responses.activate + def test_store_retrieve_url_does_not_exist(self): + responses.add( + method="GET", + url="http://test.test2", + body="Hello from testland2", + status=200, + content_type="text/plain" + ) + + nm = NotesManager(MemoryNotesStorage(), get_token) + nm.store(Note( + assoc_id="test1/11111/aaa", + subject="store_retrieve3", + sub_type="test", + note_val="this is my note 1", + link_url="test_breakage://test.test/" + )) + nm.store(Note( + assoc_id="test1/11111/bbb", + subject="store_retrieve3", + sub_type="test", + note_val="this is my note 2", + link_url="http://test.test2/" + )) + n_list = nm.retrieve(Query("test1")) + assert len(n_list) == 2 + for n in n_list: + if n.assoc_id == "test1/11111/aaa": + assert "URL lookup was unable" in n.resolved_url_value + else: + assert n.resolved_url_value.startswith("Hello from testland") + + @responses.activate + def test_store_retrieve_with_auth(self): + responses.add( + method="GET", + url="http://test.test2", + body="Hello from testland2", + status=200, + content_type="text/plain" + ) + + nm = NotesManager(MemoryNotesStorage(), get_token) + nm.store(Note( + assoc_id="test1/11111/bbb", + subject="store_retrieve3", + sub_type="test", + note_val="this is my note 2", + link_url="http://test.test2/", + is_auth_link=True + )) + n_list = nm.retrieve(Query("test1")) + assert len(n_list) == 1 + for n in n_list: + assert n.resolved_url_value == "Hello from testland2" + auth_hdr = responses.calls[0].request.headers['X-Auth-Token'] + assert 'token' == auth_hdr + + def test_note_view(self): + nm = NotesManager(MemoryNotesStorage(), get_token) + nm.store(Note( + assoc_id="test1/11111/aaa", + subject="store_retrieve_1", + sub_type="test", + note_val="this is my note 1" + )) + n_list = nm.retrieve(Query("test1")) + assert len(n_list) == 1 + nid = n_list[0].note_id + nt = n_list[0].note_timestamp + assert n_list[0].view() == { + 'assoc_id': 'test1/11111/aaa', + 'subject': 'store_retrieve_1', + 'sub_type': 'test', + 'note_val': 'this is my note 1', + 'verbosity': 1, + 'note_id': nid, + 'note_timestamp': nt, + 'resolved_url_value': None, + } diff --git a/src/bin/shipyard_airflow/tests/unit/control/test_action_validators.py b/src/bin/shipyard_airflow/tests/unit/control/test_action_validators.py index 73c0f5cc..a343d863 100644 --- a/src/bin/shipyard_airflow/tests/unit/control/test_action_validators.py +++ b/src/bin/shipyard_airflow/tests/unit/control/test_action_validators.py @@ -58,7 +58,6 @@ def get_doc_returner(style, ds_name): # if passed a name of 'defaulted' clear the section if ds_name == 'defaulted': dc.data["physical_provisioner"] = None - print(dc.__dict__) return [dc] elif doc == 'dep-strat': return [strategy] diff --git a/src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py b/src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py index 6cc2ebe3..a70e206f 100644 --- a/src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py +++ b/src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py @@ -24,6 +24,11 @@ from oslo_config import cfg import pytest import responses +from shipyard_airflow.common.notes.notes import NotesManager +from shipyard_airflow.common.notes.notes_helper import NotesHelper +from shipyard_airflow.common.notes.storage_impl_mem import ( + MemoryNotesStorage +) from shipyard_airflow.control.action import actions_api from shipyard_airflow.control.action.actions_api import ActionsResource from shipyard_airflow.control.base import ShipyardRequestContext @@ -43,6 +48,15 @@ CONF = cfg.CONF LOG = logging.getLogger(__name__) +def get_token(): + """Stub method to use for NotesHelper/NotesManager""" + return "token" + +# Notes helper that can be mocked into various objects to prevent database +# dependencies +nh = NotesHelper(NotesManager(MemoryNotesStorage(), get_token)) + + def create_req(ctx, body): '''creates a falcon request''' env = testing.create_environ( @@ -283,8 +297,6 @@ def test_get_all_actions(): action_resource.get_all_actions_db = actions_db action_resource.get_all_dag_runs_db = dag_runs_db action_resource.get_all_tasks_db = tasks_db - os.environ['DB_CONN_AIRFLOW'] = 'nothing' - os.environ['DB_CONN_SHIPYARD'] = 'nothing' result = action_resource.get_all_actions() assert len(result) == len(actions_db()) for action in result: @@ -296,6 +308,31 @@ def test_get_all_actions(): assert action['dag_status'] == 'SUCCESS' +@mock.patch('shipyard_airflow.control.action.actions_api.notes_helper', + new=nh) +def test_get_all_actions_notes(*args): + """ + Tests the main response from get all actions + """ + action_resource = ActionsResource() + action_resource.get_all_actions_db = actions_db + action_resource.get_all_dag_runs_db = dag_runs_db + action_resource.get_all_tasks_db = tasks_db + # inject some notes + nh.make_action_note('aaaaaa', "hello from aaaaaa1") + nh.make_action_note('aaaaaa', "hello from aaaaaa2") + nh.make_action_note('bbbbbb', "hello from bbbbbb") + + result = action_resource.get_all_actions() + assert len(result) == len(actions_db()) + for action in result: + if action['id'] == 'aaaaaa': + assert len(action['notes']) == 2 + if action['id'] == 'bbbbbb': + assert len(action['notes']) == 1 + assert action['notes'][0]['note_val'] == 'hello from bbbbbb' + + def _gen_action_resource_stubbed(): # TODO(bryan-strassner): mabye subclass this instead? action_resource = ActionsResource()