diff --git a/doc/source/rest.yaml b/doc/source/rest.yaml index a37a957d..00d884a9 100644 --- a/doc/source/rest.yaml +++ b/doc/source/rest.yaml @@ -315,7 +315,13 @@ POST /v1/resource_type HTTP/1.1 Content-Type: application/json - {"name": "my_custom_type"} + { + "name": "my_custom_type", + "attributes": { + "display_name": {"type": "string", "required": true}, + "prefix": {"type": "string", "required": false, "max_length": 8, "min_length": 3} + } + } - name: create-resource-type-2 request: | diff --git a/gnocchi/indexer/__init__.py b/gnocchi/indexer/__init__.py index 66b3542e..a83e4348 100644 --- a/gnocchi/indexer/__init__.py +++ b/gnocchi/indexer/__init__.py @@ -37,11 +37,6 @@ OPTS = [ _marker = object() -class ResourceType(object): - def __eq__(self, other): - return self.name == other.name - - class Resource(object): def get_metric(self, metric_name): for m in self.metrics: @@ -376,3 +371,11 @@ class IndexerDriver(object): marker=None, sorts=None): raise exceptions.NotImplementedError + + @staticmethod + def get_resource_attributes_schemas(): + raise exceptions.NotImplementedError + + @staticmethod + def get_resource_type_schema(): + raise exceptions.NotImplementedError diff --git a/gnocchi/indexer/alembic/versions/d24877c22ab0_add_attributes_to_resource_type.py b/gnocchi/indexer/alembic/versions/d24877c22ab0_add_attributes_to_resource_type.py new file mode 100644 index 00000000..dda81e50 --- /dev/null +++ b/gnocchi/indexer/alembic/versions/d24877c22ab0_add_attributes_to_resource_type.py @@ -0,0 +1,38 @@ +# Copyright 2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""Add attributes to resource_type + +Revision ID: d24877c22ab0 +Revises: 0718ed97e5b3 +Create Date: 2016-01-19 22:45:06.431190 + +""" + +from alembic import op +import sqlalchemy as sa +import sqlalchemy_utils as sa_utils + + +# revision identifiers, used by Alembic. +revision = 'd24877c22ab0' +down_revision = '0718ed97e5b3' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column("resource_type", + sa.Column('attributes', sa_utils.JSONType(),)) diff --git a/gnocchi/indexer/sqlalchemy.py b/gnocchi/indexer/sqlalchemy.py index 5534512d..02726644 100644 --- a/gnocchi/indexer/sqlalchemy.py +++ b/gnocchi/indexer/sqlalchemy.py @@ -23,7 +23,6 @@ import uuid import oslo_db.api from oslo_db import exception from oslo_db.sqlalchemy import enginefacade -from oslo_db.sqlalchemy import models from oslo_db.sqlalchemy import utils as oslo_db_utils from oslo_log import log import six @@ -34,6 +33,7 @@ from stevedore import extension from gnocchi import exceptions from gnocchi import indexer from gnocchi.indexer import sqlalchemy_base as base +from gnocchi import resource_type from gnocchi import utils Base = base.Base @@ -99,7 +99,7 @@ class ResourceClassMapper(object): tablename = resource_type.tablename # TODO(sileht): Add columns if not baseclass: - baseclass = type(str("%s_base" % tablename), (object, ), {}) + baseclass = resource_type.to_baseclass() resource_ext = type( str("%s_resource" % tablename), (baseclass, base.ResourceExtMixin, base.Resource), @@ -122,18 +122,20 @@ class ResourceClassMapper(object): mappers[tablename] = {'resource': base.Resource, 'history': base.ResourceHistory} else: - resource_type = base.ResourceType(name=ext.name, - tablename=tablename) - mappers[tablename] = self._build_class_mappers(resource_type, - ext.plugin) + rt = base.ResourceType( + name=ext.name, tablename=tablename, + attributes=resource_type.ResourceTypeAttributes()) + mappers[tablename] = self._build_class_mappers(rt, ext.plugin) return mappers def get_legacy_resource_types(self): resource_types = [] for ext in self._resources.extensions: tablename = getattr(ext.plugin, '__tablename__', ext.name) - resource_types.append(base.ResourceType(name=ext.name, - tablename=tablename)) + resource_types.append(base.ResourceType( + name=ext.name, + tablename=tablename, + attributes=resource_type.ResourceTypeAttributes())) return resource_types def get_classes(self, resource_type): @@ -250,21 +252,26 @@ class SQLAlchemyIndexer(indexer.IndexerDriver): except exception.DBDuplicateEntry: pass - def create_resource_type(self, name): + def create_resource_type(self, resource_type): # NOTE(sileht): mysql have a stupid and small length limitation on the # foreign key and index name, so we can't use the resource type name as # tablename, the limit is 64. The longest name we have is # fk__history_revision_resource_history_revision, # so 64 - 46 = 18 tablename = "rt_%s" % uuid.uuid4().hex[:15] - resource_type = ResourceType(name=name, - tablename=tablename) + resource_type = ResourceType(name=resource_type.name, + tablename=tablename, + attributes=resource_type.attributes) + + # NOTE(sileht): ensure the driver is able to store the request + # resource_type + resource_type.to_baseclass() try: with self.facade.writer() as session: session.add(resource_type) except exception.DBDuplicateEntry: - raise indexer.ResourceTypeAlreadyExists(name) + raise indexer.ResourceTypeAlreadyExists(resource_type.name) with self.facade.writer_connection() as connection: self._RESOURCE_TYPE_MANAGER.map_and_create_tables(resource_type, @@ -281,6 +288,14 @@ class SQLAlchemyIndexer(indexer.IndexerDriver): raise indexer.NoSuchResourceType(name) return resource_type + @staticmethod + def get_resource_type_schema(): + return base.RESOURCE_TYPE_SCHEMA_MANAGER + + @staticmethod + def get_resource_attributes_schemas(): + return [ext.plugin.schema() for ext in ResourceType.RESOURCE_SCHEMAS] + def list_resource_types(self): with self.facade.independent_reader() as session: return list(session.query(ResourceType).order_by( @@ -609,7 +624,7 @@ class SQLAlchemyIndexer(indexer.IndexerDriver): class Result(base.ResourceJsonifier, base.GnocchiBase): def __iter__(self): - return models.ModelIterator(self, iter(stmt.c.keys())) + return iter((key, getattr(self, key)) for key in stmt.c.keys()) sqlalchemy.orm.mapper( Result, stmt, primary_key=[stmt.c.id, stmt.c.revision], diff --git a/gnocchi/indexer/sqlalchemy_base.py b/gnocchi/indexer/sqlalchemy_base.py index 8506f8b5..6e239192 100644 --- a/gnocchi/indexer/sqlalchemy_base.py +++ b/gnocchi/indexer/sqlalchemy_base.py @@ -32,6 +32,7 @@ import sqlalchemy_utils from gnocchi import archive_policy from gnocchi import indexer +from gnocchi import resource_type from gnocchi import storage from gnocchi import utils @@ -199,7 +200,22 @@ class Metric(Base, GnocchiBase, storage.Metric): __hash__ = storage.Metric.__hash__ -class ResourceType(Base, GnocchiBase, indexer.ResourceType): +RESOURCE_TYPE_SCHEMA_MANAGER = resource_type.ResourceTypeSchemaManager( + "gnocchi.indexer.sqlalchemy.resource_type_attribute") + + +class ResourceTypeAttributes(sqlalchemy_utils.JSONType): + def process_bind_param(self, attributes, dialect): + return super(ResourceTypeAttributes, self).process_bind_param( + attributes.jsonify(), dialect) + + def process_result_value(self, value, dialect): + attributes = super(ResourceTypeAttributes, self).process_result_value( + value, dialect) + return RESOURCE_TYPE_SCHEMA_MANAGER.attributes_from_dict(attributes) + + +class ResourceType(Base, GnocchiBase, resource_type.ResourceType): __tablename__ = 'resource_type' __table_args__ = ( sqlalchemy.UniqueConstraint("tablename", @@ -210,11 +226,14 @@ class ResourceType(Base, GnocchiBase, indexer.ResourceType): name = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, nullable=False) tablename = sqlalchemy.Column(sqlalchemy.String(18), nullable=False) + attributes = sqlalchemy.Column(ResourceTypeAttributes) - def jsonify(self): - d = dict(self) - del d['tablename'] - return d + def to_baseclass(self): + cols = {} + for attr in self.attributes: + cols[attr.name] = sqlalchemy.Column(attr.satype, + nullable=not attr.required) + return type(str("%s_base" % self.tablename), (object, ), cols) class ResourceJsonifier(indexer.Resource): @@ -352,6 +371,16 @@ class ResourceHistoryExtMixin(object): ) +class HistoryModelIterator(models.ModelIterator): + def __next__(self): + # NOTE(sileht): Our custom resource attribute columns don't + # have the same name in database than in sqlalchemy model + # so remove the additional "f_" for the model name + n = six.advance_iterator(self.i) + model_attr = n[2:] if n[:2] == "f_" else n + return model_attr, getattr(self.model, n) + + class ArchivePolicyRule(Base, GnocchiBase): __tablename__ = 'archive_policy_rule' diff --git a/gnocchi/indexer/sqlalchemy_extension.py b/gnocchi/indexer/sqlalchemy_extension.py index 2f55e753..a9dd6055 100644 --- a/gnocchi/indexer/sqlalchemy_extension.py +++ b/gnocchi/indexer/sqlalchemy_extension.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -# + # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -17,6 +17,8 @@ from __future__ import absolute_import import sqlalchemy import sqlalchemy_utils +from gnocchi import resource_type + class Image(object): name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) @@ -65,3 +67,9 @@ class HostDisk(object): __tablename__ = 'host_disk' host_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) device_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=True) + + +class StringSchema(resource_type.StringSchema): + @property + def satype(self): + return sqlalchemy.String(self.max_length) diff --git a/gnocchi/resource_type.py b/gnocchi/resource_type.py new file mode 100644 index 00000000..0854544e --- /dev/null +++ b/gnocchi/resource_type.py @@ -0,0 +1,151 @@ +# -*- encoding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import re +import six +import stevedore +import voluptuous + + +INVALID_NAMES = [ + "id", "type", "metrics", + "revision", "revision_start", "revision_end", + "started_at", "ended_at", + "user_id", "project_id", + "created_by_user_id", "created_by_project_id", "get_metric" +] + +VALID_CHARS = re.compile("[a-zA-Z0-9][a-zA-Z0-9_]*") + + +class InvalidResourceAttributeName(Exception): + """Error raised when the resource attribute name is invalid.""" + def __init__(self, name): + super(InvalidResourceAttributeName, self).__init__( + "Resource attribute name %s is invalid" % str(name)) + self.name = name + + +class CommonAttributeSchema(object): + meta_schema_ext = {} + schema_ext = None + + def __init__(self, type, name, required): + if (len(name) > 63 or name in INVALID_NAMES + or not VALID_CHARS.match(name)): + raise InvalidResourceAttributeName(name) + + self.name = name + self.required = required + + @classmethod + def meta_schema(cls): + d = { + voluptuous.Required('type'): cls.typename, + voluptuous.Required('required', default=True): bool + } + d.update(cls.meta_schema_ext) + return d + + def schema(self): + if self.required: + return {self.name: self.schema_ext} + else: + return {voluptuous.Optional(self.name): self.schema_ext} + + def jsonify(self): + return {"type": self.typename, + "required": self.required} + + +class StringSchema(CommonAttributeSchema): + typename = "string" + + def __init__(self, min_length, max_length, *args, **kwargs): + super(StringSchema, self).__init__(*args, **kwargs) + self.min_length = min_length + self.max_length = max_length + + # TODO(sileht): ensure min_length <= max_length + meta_schema_ext = { + voluptuous.Required('min_length', default=0): + voluptuous.All(int, voluptuous.Range(min=0, max=255)), + voluptuous.Required('max_length', default=255): + voluptuous.All(int, voluptuous.Range(min=1, max=255)) + } + + @property + def schema_ext(self): + return voluptuous.All(six.text_type, + voluptuous.Length( + min=self.min_length, + max=self.max_length)) + + def jsonify(self): + d = super(StringSchema, self).jsonify() + d.update({"max_length": self.max_length, + "min_length": self.min_length}) + return d + + +class ResourceTypeAttributes(list): + def jsonify(self): + d = {} + for attr in self: + d[attr.name] = attr.jsonify() + return d + + +class ResourceTypeSchemaManager(stevedore.ExtensionManager): + def __init__(self, *args, **kwargs): + super(ResourceTypeSchemaManager, self).__init__(*args, **kwargs) + type_schemas = tuple([ext.plugin.meta_schema() + for ext in self.extensions]) + self._schema = voluptuous.Schema({ + "name": six.text_type, + voluptuous.Required("attributes", default={}): { + six.text_type: voluptuous.Any(*tuple(type_schemas)) + } + }) + + def __call__(self, definition): + return self._schema(definition) + + def attributes_from_dict(self, attributes): + return ResourceTypeAttributes( + self[attr["type"]].plugin(name=name, **attr) + for name, attr in attributes.items()) + + def resource_type_from_dict(self, name, attributes): + return ResourceType(name, self.attributes_from_dict(attributes)) + + +class ResourceType(object): + def __init__(self, name, attributes): + self.name = name + self.attributes = attributes + + @property + def schema(self): + schema = {} + for attr in self.attributes: + schema.update(attr.schema()) + return schema + + def __eq__(self, other): + return self.name == other.name + + def jsonify(self): + return {"name": self.name, + "attributes": self.attributes.jsonify()} diff --git a/gnocchi/rest/__init__.py b/gnocchi/rest/__init__.py index cadb496f..75c5869d 100644 --- a/gnocchi/rest/__init__.py +++ b/gnocchi/rest/__init__.py @@ -31,6 +31,7 @@ from gnocchi import aggregates from gnocchi import archive_policy from gnocchi import indexer from gnocchi import json +from gnocchi import resource_type from gnocchi import storage from gnocchi import utils @@ -813,16 +814,16 @@ class ResourceTypeController(rest.RestController): @pecan.expose('json') def get(self): try: - resource_type = pecan.request.indexer.get_resource_type(self._name) + rt = pecan.request.indexer.get_resource_type(self._name) except indexer.NoSuchResourceType as e: abort(404, e) - enforce("get resource type", resource_type) - return resource_type + enforce("get resource type", rt) + return rt @pecan.expose() def delete(self): try: - resource_type = pecan.request.indexer.get_resource_type(self._name) + pecan.request.indexer.get_resource_type(self._name) except indexer.NoSuchResourceType as e: abort(404, e) enforce("delete resource type", resource_type) @@ -833,13 +834,6 @@ class ResourceTypeController(rest.RestController): abort(400, e) -def ResourceTypeSchema(definition): - # FIXME(sileht): Add resource type attributes from the indexer - return voluptuous.Schema({ - "name": six.text_type, - })(definition) - - class ResourceTypesController(rest.RestController): @pecan.expose() @@ -848,15 +842,17 @@ class ResourceTypesController(rest.RestController): @pecan.expose('json') def post(self): - body = deserialize_and_validate(ResourceTypeSchema) + schema = pecan.request.indexer.get_resource_type_schema() + body = deserialize_and_validate(schema) + rt = schema.resource_type_from_dict(**body) enforce("create resource type", body) try: - resource_type = pecan.request.indexer.create_resource_type(**body) + rt = pecan.request.indexer.create_resource_type(rt) except indexer.ResourceTypeAlreadyExists as e: abort(409, e) - set_resp_location_hdr("/resource_type/" + resource_type.name) + set_resp_location_hdr("/resource_type/" + rt.name) pecan.response.status = 201 - return resource_type + return rt @pecan.expose('json') def get_all(self, **kwargs): @@ -1016,8 +1012,8 @@ def schema_for(resource_type): # TODO(sileht): Remove this legacy resource schema loading return RESOURCE_SCHEMA_MANAGER[resource_type].plugin else: - # TODO(sileht): Load schema from indexer - return GenericSchema + resource_type = pecan.request.indexer.get_resource_type(resource_type) + return ResourceSchema(resource_type.schema) def ResourceID(value): diff --git a/gnocchi/tests/gabbi/gabbits/resource_type.yaml b/gnocchi/tests/gabbi/gabbits/resource_type.yaml index a910dad8..f02d1017 100644 --- a/gnocchi/tests/gabbi/gabbits/resource_type.yaml +++ b/gnocchi/tests/gabbi/gabbits/resource_type.yaml @@ -23,6 +23,28 @@ tests: content-type: application/json status: 403 + - name: post resource type bad string + url: /v1/resource_type + method: post + request_headers: + x-roles: admin + content-type: application/json + data: + name: my_custom_resource + attributes: + foo: + type: string + max_length: 32 + min_length: 5 + noexist: foo + status: 400 + response_strings: + # NOTE(sileht): We would prefer to have a better message but voluptuous seems a bit lost when + # an Any have many dict with the same key, here "type" + # - "Invalid input: extra keys not allowed @ data[u'attributes'][u'foo'][u'noexist']" + # - "Invalid input: not a valid value for dictionary value @ data[u'attributes'][u'foo'][u'type']" + - "Invalid input:" + - name: post resource type url: /v1/resource_type method: post @@ -31,9 +53,29 @@ tests: content-type: application/json data: name: my_custom_resource + attributes: + name: + type: string + required: true + max_length: 5 + min_length: 2 + foobar: + type: string + required: false status: 201 response_json_paths: $.name: my_custom_resource + $.attributes: + name: + type: string + required: True + max_length: 5 + min_length: 2 + foobar: + type: string + required: False + max_length: 255 + min_length: 0 response_headers: location: $SCHEME://$NETLOC/v1/resource_type/my_custom_resource @@ -54,6 +96,23 @@ tests: method: DELETE status: 403 + - name: post invalid resource + url: /v1/resource/my_custom_resource + method: post + request_headers: + x-user-id: 0fbb2314-8461-4b1a-8013-1fc22f6afc9c + x-project-id: f3d41b77-0cc1-4f0b-b94a-1d5be9c0e3ea + content-type: application/json + data: + id: d11edfca-4393-4fda-b94d-b05a3a1b3747 + name: toolong!!! + foobar: what + status: 400 + response_strings: + # split to not match the u' in py2 + - "Invalid input: length of value must be at most 5 for dictionary value @ data[" + - "'name']" + - name: post custom resource url: /v1/resource/my_custom_resource method: post @@ -63,7 +122,50 @@ tests: content-type: application/json data: id: d11edfca-4393-4fda-b94d-b05a3a1b3747 + name: bar + foobar: what status: 201 + response_json_paths: + $.id: d11edfca-4393-4fda-b94d-b05a3a1b3747 + $.name: bar + $.foobar: what + + - name: patch custom resource + url: /v1/resource/my_custom_resource/d11edfca-4393-4fda-b94d-b05a3a1b3747 + method: patch + request_headers: + x-user-id: 0fbb2314-8461-4b1a-8013-1fc22f6afc9c + x-project-id: f3d41b77-0cc1-4f0b-b94a-1d5be9c0e3ea + content-type: application/json + data: + name: foo + status: 200 + response_json_paths: + $.id: d11edfca-4393-4fda-b94d-b05a3a1b3747 + $.name: foo + $.foobar: what + + - name: get resource + url: /v1/resource/my_custom_resource/d11edfca-4393-4fda-b94d-b05a3a1b3747 + request_headers: + content-type: application/json + response_json_paths: + $.id: d11edfca-4393-4fda-b94d-b05a3a1b3747 + $.name: foo + $.foobar: what + + - name: list resource history + url: /v1/resource/my_custom_resource/d11edfca-4393-4fda-b94d-b05a3a1b3747/history?sort=revision_end:asc-nullslast + request_headers: + content-type: application/json + response_json_paths: + $.`len`: 2 + $[0].id: d11edfca-4393-4fda-b94d-b05a3a1b3747 + $[0].name: bar + $[0].foobar: what + $[1].id: d11edfca-4393-4fda-b94d-b05a3a1b3747 + $[1].name: foo + $[1].foobar: what - name: delete in use resource_type url: /v1/resource_type/my_custom_resource diff --git a/gnocchi/tests/test_indexer.py b/gnocchi/tests/test_indexer.py index 5fc065ef..2f049ded 100644 --- a/gnocchi/tests/test_indexer.py +++ b/gnocchi/tests/test_indexer.py @@ -974,15 +974,28 @@ class TestIndexerDriver(tests_base.TestCase): self.assertNotIn(e1, [m.id for m in metrics]) def test_resource_type_crud(self): + mgr = self.index.get_resource_type_schema() + rtype = mgr.resource_type_from_dict("indexer_test", { + "col1": {"type": "string", "required": True, + "min_length": 2, "max_length": 15} + }) + # Create - self.index.create_resource_type("indexer_test") + self.index.create_resource_type(rtype) self.assertRaises(indexer.ResourceTypeAlreadyExists, self.index.create_resource_type, - "indexer_test") + rtype) - # Get and List + # Get rtype = self.index.get_resource_type("indexer_test") self.assertEqual("indexer_test", rtype.name) + self.assertEqual(1, len(rtype.attributes)) + self.assertEqual("col1", rtype.attributes[0].name) + self.assertEqual("string", rtype.attributes[0].typename) + self.assertEqual(15, rtype.attributes[0].max_length) + self.assertEqual(2, rtype.attributes[0].min_length) + + # List rtypes = self.index.list_resource_types() for rtype in rtypes: if rtype.name == "indexer_test": @@ -994,9 +1007,11 @@ class TestIndexerDriver(tests_base.TestCase): rid = uuid.uuid4() self.index.create_resource("indexer_test", rid, str(uuid.uuid4()), - str(uuid.uuid4())) + str(uuid.uuid4()), + col1="col1_value") r = self.index.get_resource("indexer_test", rid) self.assertEqual("indexer_test", r.type) + self.assertEqual("col1_value", r.col1) # Deletion self.assertRaises(indexer.ResourceTypeInUse, diff --git a/setup.cfg b/setup.cfg index 9654e7d8..bb5e7803 100644 --- a/setup.cfg +++ b/setup.cfg @@ -85,6 +85,9 @@ data_files = etc/gnocchi = etc/gnocchi/* [entry_points] +gnocchi.indexer.sqlalchemy.resource_type_attribute = + string = gnocchi.indexer.sqlalchemy_extension:StringSchema + gnocchi.indexer.resources = generic = gnocchi.indexer.sqlalchemy_base:Resource instance = gnocchi.indexer.sqlalchemy_extension:Instance