# Copyright (c) 2015 Mirantis, Inc. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. """Implementation of SQLAlchemy backend.""" import base64 import hashlib import sys import uuid from oslo_config import cfg from oslo_db import options as db_options from oslo_db.sqlalchemy import session as db_session from oslo_log import log from refstack.api import constants as api_const from refstack.db.sqlalchemy import models CONF = cfg.CONF _FACADE = None LOG = log.getLogger(__name__) db_options.set_defaults(cfg.CONF) class NotFound(Exception): """Raise if item not found in db.""" pass class Duplication(Exception): """Raise if unique constraint violates.""" pass def _create_facade_lazily(): """Create DB facade lazily.""" global _FACADE if _FACADE is None: _FACADE = db_session.EngineFacade.from_config(CONF) return _FACADE def get_engine(): """Get DB engine.""" facade = _create_facade_lazily() return facade.get_engine() def get_session(**kwargs): """Get DB session.""" facade = _create_facade_lazily() return facade.get_session(**kwargs) def get_backend(): """The backend is this module itself.""" return sys.modules[__name__] def _to_dict(sqlalchemy_object, allowed_keys=None): if isinstance(sqlalchemy_object, list): return [_to_dict(obj, allowed_keys=allowed_keys) for obj in sqlalchemy_object] if (hasattr(sqlalchemy_object, 'keys') and hasattr(sqlalchemy_object, 'index')): return {key: getattr(sqlalchemy_object, key) for key in sqlalchemy_object.keys()} if hasattr(sqlalchemy_object, 'default_allowed_keys'): items = sqlalchemy_object.iteritems() if not allowed_keys: allowed_keys = sqlalchemy_object.default_allowed_keys if allowed_keys: items = filter(lambda item: item[0] in allowed_keys, items) result = {} for key, value in items: if key in sqlalchemy_object.metadata_keys: result[key] = { item.get(sqlalchemy_object.metadata_keys[key]['key']): item.get(sqlalchemy_object.metadata_keys[key]['value']) for item in value} elif hasattr(value, 'default_allowed_keys'): result[key] = _to_dict(value) elif (isinstance(value, list) and value and hasattr(value[0], 'default_allowed_keys')): result[key] = [_to_dict(item) for item in value] else: result[key] = value return result if hasattr(sqlalchemy_object, 'all'): return _to_dict(sqlalchemy_object.all()) return sqlalchemy_object def store_test_results(results): """Store test results.""" test = models.Test() test_id = str(uuid.uuid4()) test.id = test_id test.cpid = results.get('cpid') test.duration_seconds = results.get('duration_seconds') test.product_version_id = results.get('product_version_id') session = get_session() with session.begin(): for result in results.get('results', []): test_result = models.TestResults() test_result.test_id = test_id test_result.name = result['name'] test_result.uuid = result.get('uuid', None) test.results.append(test_result) for k, v in results.get('meta', {}).items(): meta = models.TestMeta() meta.meta_key, meta.value = k, v test.meta.append(meta) test.save(session) return test_id def get_test_result(test_id, allowed_keys=None): """Get test info.""" session = get_session() test_info = session.query(models.Test). \ filter_by(id=test_id). \ first() if not test_info: raise NotFound('Test result %s not found' % test_id) return _to_dict(test_info, allowed_keys) def delete_test_result(test_id): """Delete test information from the database.""" session = get_session() with session.begin(): test = session.query(models.Test).filter_by(id=test_id).first() if test: session.query(models.TestMeta) \ .filter_by(test_id=test_id).delete() session.query(models.TestResults) \ .filter_by(test_id=test_id).delete() session.delete(test) else: raise NotFound('Test result %s not found' % test_id) def update_test_result(test_info): """Update test from the given test_info dictionary.""" session = get_session() _id = test_info.get('id') test = session.query(models.Test).filter_by(id=_id).first() if test is None: raise NotFound('Test result with id %s not found' % _id) keys = ['product_version_id', 'verification_status'] for key in keys: if key in test_info: setattr(test, key, test_info[key]) with session.begin(): test.save(session=session) return _to_dict(test) def get_test_result_meta_key(test_id, key, default=None): """Get metadata value related to specified test run.""" session = get_session() meta_item = session.query(models.TestMeta). \ filter_by(test_id=test_id). \ filter_by(meta_key=key). \ first() value = meta_item.value if meta_item else default return value def save_test_result_meta_item(test_id, key, value): """Store or update item value related to specified test run.""" session = get_session() meta_item = (session.query(models.TestMeta) .filter_by(test_id=test_id) .filter_by(meta_key=key).first() or models.TestMeta()) meta_item.test_id = test_id meta_item.meta_key = key meta_item.value = value with session.begin(): meta_item.save(session) def delete_test_result_meta_item(test_id, key): """Delete metadata item related to specified test run.""" session = get_session() meta_item = session.query(models.TestMeta). \ filter_by(test_id=test_id). \ filter_by(meta_key=key). \ first() if meta_item: with session.begin(): session.delete(meta_item) else: raise NotFound('Metadata key %s ' 'not found for test run %s' % (key, test_id)) def get_test_results(test_id): """Get test results.""" session = get_session() results = session.query(models.TestResults). \ filter_by(test_id=test_id). \ all() return [_to_dict(result) for result in results] def _apply_filters_for_query(query, filters): """Apply filters for DB query.""" start_date = filters.get(api_const.START_DATE) if start_date: query = query.filter(models.Test.created_at >= start_date) end_date = filters.get(api_const.END_DATE) if end_date: query = query.filter(models.Test.created_at <= end_date) cpid = filters.get(api_const.CPID) if cpid: query = query.filter(models.Test.cpid == cpid) verification_status = filters.get(api_const.VERIFICATION_STATUS) if verification_status: query = query.filter(models.Test.verification_status == verification_status) if api_const.PRODUCT_ID in filters: query = (query .join(models.ProductVersion) .filter(models.ProductVersion.product_id == filters[api_const.PRODUCT_ID])) all_product_tests = filters.get(api_const.ALL_PRODUCT_TESTS) signed = api_const.SIGNED in filters # If we only want to get the user's test results. if signed: query = (query .join(models.Test.meta) .filter(models.TestMeta.meta_key == api_const.USER) .filter(models.TestMeta.value == filters[api_const.OPENID]) ) elif not all_product_tests: # Get all non-signed (aka anonymously uploaded) test results # along with signed but shared test results. signed_results = (query.session .query(models.TestMeta.test_id) .filter_by(meta_key=api_const.USER)) shared_results = (query.session .query(models.TestMeta.test_id) .filter_by(meta_key=api_const.SHARED_TEST_RUN)) query = (query.filter(models.Test.id.notin_(signed_results)) .union(query.filter(models.Test.id.in_(shared_results)))) return query def get_test_result_records(page, per_page, filters): """Get page with list of test records.""" session = get_session() query = session.query(models.Test) query = _apply_filters_for_query(query, filters) results = query.order_by(models.Test.created_at.desc()). \ offset(per_page * (page - 1)). \ limit(per_page).all() return _to_dict(results) def get_test_result_records_count(filters): """Get total test records count.""" session = get_session() query = session.query(models.Test.id) records_count = _apply_filters_for_query(query, filters).count() return records_count def user_get(user_openid): """Get user info by openid.""" session = get_session() user = session.query(models.User).filter_by(openid=user_openid).first() if user is None: raise NotFound('User with OpenID %s not found' % user_openid) return user def user_save(user_info): """Create user DB record if it exists, otherwise record will be updated.""" try: user = user_get(user_info['openid']) except NotFound: user = models.User() session = get_session() with session.begin(): user.update(user_info) user.save(session=session) return user def get_pubkey(key): """Get the pubkey info corresponding to the given public key. The md5 hash of the key is used for the query for quicker lookups. """ session = get_session() md5_hash = hashlib.md5(base64.b64decode(key)).hexdigest() pubkeys = session.query(models.PubKey).filter_by(md5_hash=md5_hash).all() if len(pubkeys) == 1: return pubkeys[0] elif len(pubkeys) > 1: for pubkey in pubkeys: if pubkey['pubkey'] == key: return pubkey return None def store_pubkey(pubkey_info): """Store public key in to DB.""" pubkey = models.PubKey() pubkey.openid = pubkey_info['openid'] pubkey.format = pubkey_info['format'] pubkey.pubkey = pubkey_info['pubkey'] pubkey.md5_hash = hashlib.md5( base64.b64decode( pubkey_info['pubkey'] ) ).hexdigest() pubkey.comment = pubkey_info['comment'] session = get_session() with session.begin(): pubkeys_collision = (session. query(models.PubKey). filter_by(md5_hash=pubkey.md5_hash). filter_by(pubkey=pubkey.pubkey).all()) if not pubkeys_collision: pubkey.save(session) else: raise Duplication('Public key already exists.') return pubkey.id def delete_pubkey(id): """Delete public key from DB.""" session = get_session() with session.begin(): key = session.query(models.PubKey).filter_by(id=id).first() session.delete(key) def get_user_pubkeys(user_openid): """Get public pubkeys for specified user.""" session = get_session() pubkeys = session.query(models.PubKey).filter_by(openid=user_openid).all() return _to_dict(pubkeys) def add_user_to_group(user_openid, group_id, created_by_user): """Add specified user to specified group.""" item = models.UserToGroup() session = get_session() with session.begin(): item.user_openid = user_openid item.group_id = group_id item.created_by_user = created_by_user item.save(session=session) def remove_user_from_group(user_openid, group_id): """Remove specified user from specified group.""" session = get_session() with session.begin(): (session.query(models.UserToGroup). filter_by(user_openid=user_openid). filter_by(group_id=group_id). delete(synchronize_session=False)) def add_organization(organization_info, creator): """Add organization.""" session = get_session() with session.begin(): group = models.Group() group.name = 'Group for %s' % organization_info['name'] group.save(session=session) group_id = group.id item = models.UserToGroup() item.user_openid = creator item.group_id = group_id item.created_by_user = creator item.save(session=session) organization = models.Organization() organization.type = organization_info.get( 'type', api_const.PRIVATE_VENDOR) organization.name = organization_info['name'] organization.description = organization_info.get('description') organization.group_id = group_id organization.created_by_user = creator organization.properties = organization_info.get('properties') organization.save(session=session) return _to_dict(organization) def update_organization(organization_info): """Update organization.""" session = get_session() _id = organization_info['id'] organization = (session.query(models.Organization). filter_by(id=_id).first()) if organization is None: raise NotFound('Organization with id %s not found' % _id) with session.begin(): organization.type = organization_info.get( 'type', organization.type) organization.name = organization_info.get( 'name', organization.name) organization.description = organization_info.get( 'description', organization.description) organization.properties = organization_info.get( 'properties', organization.properties) organization.save(session=session) return _to_dict(organization) def get_organization(organization_id, allowed_keys=None): """Get organization by id.""" session = get_session() organization = (session.query(models.Organization). filter_by(id=organization_id).first()) if organization is None: raise NotFound('Organization with id %s not found' % organization_id) return _to_dict(organization, allowed_keys=allowed_keys) def delete_organization(organization_id): """delete organization by id.""" session = get_session() with session.begin(): product_ids = (session .query(models.Product.id) .filter_by(organization_id=organization_id)) (session.query(models.ProductVersion). filter(models.ProductVersion.product_id.in_(product_ids)). delete(synchronize_session=False)) (session.query(models.Product). filter_by(organization_id=organization_id). delete(synchronize_session=False)) (session.query(models.Organization). filter_by(id=organization_id). delete(synchronize_session=False)) def add_product(product_info, creator): """Add product.""" product = models.Product() product.id = str(uuid.uuid4()) product.type = product_info['type'] product.product_type = product_info['product_type'] product.product_ref_id = product_info.get('product_ref_id') product.name = product_info['name'] product.description = product_info.get('description') product.organization_id = product_info['organization_id'] product.created_by_user = creator product.public = product_info.get('public', False) product.properties = product_info.get('properties') session = get_session() with session.begin(): product.save(session=session) product_version = models.ProductVersion() product_version.created_by_user = creator product_version.version = product_info.get('version') product_version.product_id = product.id product_version.save(session=session) return _to_dict(product) def update_product(product_info): """Update product by id.""" session = get_session() _id = product_info.get('id') product = session.query(models.Product).filter_by(id=_id).first() if product is None: raise NotFound('Product with id %s not found' % _id) keys = ['name', 'description', 'product_ref_id', 'public', 'properties'] for key in keys: if key in product_info: setattr(product, key, product_info[key]) with session.begin(): product.save(session=session) return _to_dict(product) def get_product(id, allowed_keys=None): """Get product by id.""" session = get_session() product = session.query(models.Product).filter_by(id=id).first() if product is None: raise NotFound('Product with id "%s" not found' % id) return _to_dict(product, allowed_keys=allowed_keys) def delete_product(id): """delete product by id.""" session = get_session() with session.begin(): (session.query(models.ProductVersion) .filter_by(product_id=id) .delete(synchronize_session=False)) (session.query(models.Product).filter_by(id=id). delete(synchronize_session=False)) def get_foundation_users(): """Get users' openid-s that belong to group of foundation.""" session = get_session() organization = ( session.query(models.Organization.group_id) .filter_by(type=api_const.FOUNDATION).first()) if organization is None: LOG.warning('Foundation organization record not found in DB.') return [] group_id = organization.group_id users = (session.query(models.UserToGroup.user_openid). filter_by(group_id=group_id)) return [user.user_openid for user in users] def get_organization_users(organization_id): """Get users that belong to group of organization.""" session = get_session() organization = (session.query(models.Organization.group_id) .filter_by(id=organization_id).first()) if organization is None: raise NotFound('Organization with id %s is not found' % organization_id) group_id = organization.group_id users = (session.query(models.UserToGroup, models.User) .join(models.User, models.User.openid == models.UserToGroup.user_openid) .filter(models.UserToGroup.group_id == group_id)) keys = ['openid', 'fullname', 'email'] return {item[1].openid: _to_dict(item[1], allowed_keys=keys) for item in users} def get_organizations(allowed_keys=None): """Get all organizations.""" session = get_session() items = ( session.query(models.Organization) .order_by(models.Organization.created_at.desc()).all()) return _to_dict(items, allowed_keys=allowed_keys) def get_organizations_by_types(types, allowed_keys=None): """Get organization by list of types.""" session = get_session() items = ( session.query(models.Organization) .filter(models.Organization.type.in_(types)) .order_by(models.Organization.created_at.desc()).all()) return _to_dict(items, allowed_keys=allowed_keys) def get_organizations_by_user(user_openid, allowed_keys=None): """Get organizations for specified user.""" session = get_session() items = ( session.query(models.Organization, models.Group, models.UserToGroup) .join(models.Group, models.Group.id == models.Organization.group_id) .join(models.UserToGroup, models.Group.id == models.UserToGroup.group_id) .filter(models.UserToGroup.user_openid == user_openid) .order_by(models.Organization.created_at.desc()).all()) items = [item[0] for item in items] return _to_dict(items, allowed_keys=allowed_keys) def get_products(allowed_keys=None, filters=None): """Get products based on passed in filters.""" if filters is None: filters = {} expected_filters = ['public', 'organization_id'] filter_args = {} for key, value in filters.items(): if key not in expected_filters: raise Exception('Unknown filter key "%s"' % key) filter_args[key] = value session = get_session() query = session.query(models.Product) if filter_args: query = query.filter_by(**filter_args) items = query.order_by(models.Product.created_at.desc()).all() return _to_dict(items, allowed_keys=allowed_keys) def get_products_by_user(user_openid, allowed_keys=None, filters=None): """Get products that a user can manage.""" if filters is None: filters = {} session = get_session() query = ( session.query(models.Product, models.Organization, models.Group, models.UserToGroup) .join(models.Organization, models.Organization.id == models.Product.organization_id) .join(models.Group, models.Group.id == models.Organization.group_id) .join(models.UserToGroup, models.Group.id == models.UserToGroup.group_id) .filter(models.UserToGroup.user_openid == user_openid)) expected_filters = ['organization_id'] for key, value in filters.items(): if key not in expected_filters: raise Exception('Unknown filter key "%s"' % key) query = query.filter(getattr(models.Product, key) == filters[key]) items = query.order_by(models.Organization.created_at.desc()).all() items = [item[0] for item in items] return _to_dict(items, allowed_keys=allowed_keys) def get_product_by_version(product_version_id, allowed_keys=None): """Get product info from a product version ID.""" session = get_session() product = (session.query(models.Product).join(models.ProductVersion) .filter(models.ProductVersion.id == product_version_id).first()) return _to_dict(product, allowed_keys=allowed_keys) def get_product_version(product_version_id, allowed_keys=None): """Get details of a specific version given the id.""" session = get_session() version = ( session.query(models.ProductVersion) .filter_by(id=product_version_id).first() ) if version is None: raise NotFound('Version with id "%s" not found' % product_version_id) return _to_dict(version, allowed_keys=allowed_keys) def get_product_version_by_cpid(cpid, allowed_keys=None): """Get a product version given a cloud provider id.""" session = get_session() version = ( session.query(models.ProductVersion) .filter_by(cpid=cpid).all() ) return _to_dict(version, allowed_keys=allowed_keys) def get_product_versions(product_id, allowed_keys=None): """Get all versions for a product.""" session = get_session() version_info = ( session.query(models.ProductVersion) .filter_by(product_id=product_id).all() ) return _to_dict(version_info, allowed_keys=allowed_keys) def add_product_version(product_id, version, creator, cpid, allowed_keys=None): """Add a new product version.""" product_version = models.ProductVersion() product_version.created_by_user = creator product_version.version = version product_version.product_id = product_id product_version.cpid = cpid session = get_session() with session.begin(): product_version.save(session=session) return _to_dict(product_version, allowed_keys=allowed_keys) def update_product_version(product_version_info): """Update product version from product_info_version dictionary.""" session = get_session() _id = product_version_info.get('id') version = session.query(models.ProductVersion).filter_by(id=_id).first() if version is None: raise NotFound('Product version with id %s not found' % _id) # Only allow updating cpid. keys = ['cpid'] for key in keys: if key in product_version_info: setattr(version, key, product_version_info[key]) with session.begin(): version.save(session=session) return _to_dict(version) def delete_product_version(product_version_id): """Delete a product version.""" session = get_session() with session.begin(): (session.query(models.ProductVersion).filter_by(id=product_version_id). delete(synchronize_session=False))