Implements resource attribute string

This change allows to create resource attributes
of type string.

The choosen solution is that the indexer is responsible
to providing voluptuous schemas and build resource columns
according the the voluptuous schema that it proposes.

Another alternative could be that rest API provides
jsonschema for each times and indexer is only
responsible for the schema to sql column convertion.

Blueprint resource-type-rest-api
Change-Id: I7877b6ea97dc70f3629e63abe5ef1ddf61d200b3
This commit is contained in:
Mehdi Abaakouk 2016-01-19 22:55:59 +01:00
parent 420b1c424b
commit 65a9ac9941
11 changed files with 412 additions and 46 deletions

View File

@ -315,7 +315,13 @@
POST /v1/resource_type HTTP/1.1
Content-Type: application/json
{"name": "my_custom_type"}
{
"name": "my_custom_type",
"attributes": {
"display_name": {"type": "string", "required": true},
"prefix": {"type": "string", "required": false, "max_length": 8, "min_length": 3}
}
}
- name: create-resource-type-2
request: |

View File

@ -37,11 +37,6 @@ OPTS = [
_marker = object()
class ResourceType(object):
def __eq__(self, other):
return self.name == other.name
class Resource(object):
def get_metric(self, metric_name):
for m in self.metrics:
@ -376,3 +371,11 @@ class IndexerDriver(object):
marker=None,
sorts=None):
raise exceptions.NotImplementedError
@staticmethod
def get_resource_attributes_schemas():
raise exceptions.NotImplementedError
@staticmethod
def get_resource_type_schema():
raise exceptions.NotImplementedError

View File

@ -0,0 +1,38 @@
# Copyright 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
"""Add attributes to resource_type
Revision ID: d24877c22ab0
Revises: 0718ed97e5b3
Create Date: 2016-01-19 22:45:06.431190
"""
from alembic import op
import sqlalchemy as sa
import sqlalchemy_utils as sa_utils
# revision identifiers, used by Alembic.
revision = 'd24877c22ab0'
down_revision = '0718ed97e5b3'
branch_labels = None
depends_on = None
def upgrade():
op.add_column("resource_type",
sa.Column('attributes', sa_utils.JSONType(),))

View File

@ -23,7 +23,6 @@ import uuid
import oslo_db.api
from oslo_db import exception
from oslo_db.sqlalchemy import enginefacade
from oslo_db.sqlalchemy import models
from oslo_db.sqlalchemy import utils as oslo_db_utils
from oslo_log import log
import six
@ -34,6 +33,7 @@ from stevedore import extension
from gnocchi import exceptions
from gnocchi import indexer
from gnocchi.indexer import sqlalchemy_base as base
from gnocchi import resource_type
from gnocchi import utils
Base = base.Base
@ -99,7 +99,7 @@ class ResourceClassMapper(object):
tablename = resource_type.tablename
# TODO(sileht): Add columns
if not baseclass:
baseclass = type(str("%s_base" % tablename), (object, ), {})
baseclass = resource_type.to_baseclass()
resource_ext = type(
str("%s_resource" % tablename),
(baseclass, base.ResourceExtMixin, base.Resource),
@ -122,18 +122,20 @@ class ResourceClassMapper(object):
mappers[tablename] = {'resource': base.Resource,
'history': base.ResourceHistory}
else:
resource_type = base.ResourceType(name=ext.name,
tablename=tablename)
mappers[tablename] = self._build_class_mappers(resource_type,
ext.plugin)
rt = base.ResourceType(
name=ext.name, tablename=tablename,
attributes=resource_type.ResourceTypeAttributes())
mappers[tablename] = self._build_class_mappers(rt, ext.plugin)
return mappers
def get_legacy_resource_types(self):
resource_types = []
for ext in self._resources.extensions:
tablename = getattr(ext.plugin, '__tablename__', ext.name)
resource_types.append(base.ResourceType(name=ext.name,
tablename=tablename))
resource_types.append(base.ResourceType(
name=ext.name,
tablename=tablename,
attributes=resource_type.ResourceTypeAttributes()))
return resource_types
def get_classes(self, resource_type):
@ -250,21 +252,26 @@ class SQLAlchemyIndexer(indexer.IndexerDriver):
except exception.DBDuplicateEntry:
pass
def create_resource_type(self, name):
def create_resource_type(self, resource_type):
# NOTE(sileht): mysql have a stupid and small length limitation on the
# foreign key and index name, so we can't use the resource type name as
# tablename, the limit is 64. The longest name we have is
# fk_<tablename>_history_revision_resource_history_revision,
# so 64 - 46 = 18
tablename = "rt_%s" % uuid.uuid4().hex[:15]
resource_type = ResourceType(name=name,
tablename=tablename)
resource_type = ResourceType(name=resource_type.name,
tablename=tablename,
attributes=resource_type.attributes)
# NOTE(sileht): ensure the driver is able to store the request
# resource_type
resource_type.to_baseclass()
try:
with self.facade.writer() as session:
session.add(resource_type)
except exception.DBDuplicateEntry:
raise indexer.ResourceTypeAlreadyExists(name)
raise indexer.ResourceTypeAlreadyExists(resource_type.name)
with self.facade.writer_connection() as connection:
self._RESOURCE_TYPE_MANAGER.map_and_create_tables(resource_type,
@ -281,6 +288,14 @@ class SQLAlchemyIndexer(indexer.IndexerDriver):
raise indexer.NoSuchResourceType(name)
return resource_type
@staticmethod
def get_resource_type_schema():
return base.RESOURCE_TYPE_SCHEMA_MANAGER
@staticmethod
def get_resource_attributes_schemas():
return [ext.plugin.schema() for ext in ResourceType.RESOURCE_SCHEMAS]
def list_resource_types(self):
with self.facade.independent_reader() as session:
return list(session.query(ResourceType).order_by(
@ -609,7 +624,7 @@ class SQLAlchemyIndexer(indexer.IndexerDriver):
class Result(base.ResourceJsonifier, base.GnocchiBase):
def __iter__(self):
return models.ModelIterator(self, iter(stmt.c.keys()))
return iter((key, getattr(self, key)) for key in stmt.c.keys())
sqlalchemy.orm.mapper(
Result, stmt, primary_key=[stmt.c.id, stmt.c.revision],

View File

@ -32,6 +32,7 @@ import sqlalchemy_utils
from gnocchi import archive_policy
from gnocchi import indexer
from gnocchi import resource_type
from gnocchi import storage
from gnocchi import utils
@ -199,7 +200,22 @@ class Metric(Base, GnocchiBase, storage.Metric):
__hash__ = storage.Metric.__hash__
class ResourceType(Base, GnocchiBase, indexer.ResourceType):
RESOURCE_TYPE_SCHEMA_MANAGER = resource_type.ResourceTypeSchemaManager(
"gnocchi.indexer.sqlalchemy.resource_type_attribute")
class ResourceTypeAttributes(sqlalchemy_utils.JSONType):
def process_bind_param(self, attributes, dialect):
return super(ResourceTypeAttributes, self).process_bind_param(
attributes.jsonify(), dialect)
def process_result_value(self, value, dialect):
attributes = super(ResourceTypeAttributes, self).process_result_value(
value, dialect)
return RESOURCE_TYPE_SCHEMA_MANAGER.attributes_from_dict(attributes)
class ResourceType(Base, GnocchiBase, resource_type.ResourceType):
__tablename__ = 'resource_type'
__table_args__ = (
sqlalchemy.UniqueConstraint("tablename",
@ -210,11 +226,14 @@ class ResourceType(Base, GnocchiBase, indexer.ResourceType):
name = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True,
nullable=False)
tablename = sqlalchemy.Column(sqlalchemy.String(18), nullable=False)
attributes = sqlalchemy.Column(ResourceTypeAttributes)
def jsonify(self):
d = dict(self)
del d['tablename']
return d
def to_baseclass(self):
cols = {}
for attr in self.attributes:
cols[attr.name] = sqlalchemy.Column(attr.satype,
nullable=not attr.required)
return type(str("%s_base" % self.tablename), (object, ), cols)
class ResourceJsonifier(indexer.Resource):
@ -352,6 +371,16 @@ class ResourceHistoryExtMixin(object):
)
class HistoryModelIterator(models.ModelIterator):
def __next__(self):
# NOTE(sileht): Our custom resource attribute columns don't
# have the same name in database than in sqlalchemy model
# so remove the additional "f_" for the model name
n = six.advance_iterator(self.i)
model_attr = n[2:] if n[:2] == "f_" else n
return model_attr, getattr(self.model, n)
class ArchivePolicyRule(Base, GnocchiBase):
__tablename__ = 'archive_policy_rule'

View File

@ -1,5 +1,5 @@
# -*- encoding: utf-8 -*-
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
@ -17,6 +17,8 @@ from __future__ import absolute_import
import sqlalchemy
import sqlalchemy_utils
from gnocchi import resource_type
class Image(object):
name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
@ -65,3 +67,9 @@ class HostDisk(object):
__tablename__ = 'host_disk'
host_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
device_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
class StringSchema(resource_type.StringSchema):
@property
def satype(self):
return sqlalchemy.String(self.max_length)

151
gnocchi/resource_type.py Normal file
View File

@ -0,0 +1,151 @@
# -*- encoding: utf-8 -*-
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import re
import six
import stevedore
import voluptuous
INVALID_NAMES = [
"id", "type", "metrics",
"revision", "revision_start", "revision_end",
"started_at", "ended_at",
"user_id", "project_id",
"created_by_user_id", "created_by_project_id", "get_metric"
]
VALID_CHARS = re.compile("[a-zA-Z0-9][a-zA-Z0-9_]*")
class InvalidResourceAttributeName(Exception):
"""Error raised when the resource attribute name is invalid."""
def __init__(self, name):
super(InvalidResourceAttributeName, self).__init__(
"Resource attribute name %s is invalid" % str(name))
self.name = name
class CommonAttributeSchema(object):
meta_schema_ext = {}
schema_ext = None
def __init__(self, type, name, required):
if (len(name) > 63 or name in INVALID_NAMES
or not VALID_CHARS.match(name)):
raise InvalidResourceAttributeName(name)
self.name = name
self.required = required
@classmethod
def meta_schema(cls):
d = {
voluptuous.Required('type'): cls.typename,
voluptuous.Required('required', default=True): bool
}
d.update(cls.meta_schema_ext)
return d
def schema(self):
if self.required:
return {self.name: self.schema_ext}
else:
return {voluptuous.Optional(self.name): self.schema_ext}
def jsonify(self):
return {"type": self.typename,
"required": self.required}
class StringSchema(CommonAttributeSchema):
typename = "string"
def __init__(self, min_length, max_length, *args, **kwargs):
super(StringSchema, self).__init__(*args, **kwargs)
self.min_length = min_length
self.max_length = max_length
# TODO(sileht): ensure min_length <= max_length
meta_schema_ext = {
voluptuous.Required('min_length', default=0):
voluptuous.All(int, voluptuous.Range(min=0, max=255)),
voluptuous.Required('max_length', default=255):
voluptuous.All(int, voluptuous.Range(min=1, max=255))
}
@property
def schema_ext(self):
return voluptuous.All(six.text_type,
voluptuous.Length(
min=self.min_length,
max=self.max_length))
def jsonify(self):
d = super(StringSchema, self).jsonify()
d.update({"max_length": self.max_length,
"min_length": self.min_length})
return d
class ResourceTypeAttributes(list):
def jsonify(self):
d = {}
for attr in self:
d[attr.name] = attr.jsonify()
return d
class ResourceTypeSchemaManager(stevedore.ExtensionManager):
def __init__(self, *args, **kwargs):
super(ResourceTypeSchemaManager, self).__init__(*args, **kwargs)
type_schemas = tuple([ext.plugin.meta_schema()
for ext in self.extensions])
self._schema = voluptuous.Schema({
"name": six.text_type,
voluptuous.Required("attributes", default={}): {
six.text_type: voluptuous.Any(*tuple(type_schemas))
}
})
def __call__(self, definition):
return self._schema(definition)
def attributes_from_dict(self, attributes):
return ResourceTypeAttributes(
self[attr["type"]].plugin(name=name, **attr)
for name, attr in attributes.items())
def resource_type_from_dict(self, name, attributes):
return ResourceType(name, self.attributes_from_dict(attributes))
class ResourceType(object):
def __init__(self, name, attributes):
self.name = name
self.attributes = attributes
@property
def schema(self):
schema = {}
for attr in self.attributes:
schema.update(attr.schema())
return schema
def __eq__(self, other):
return self.name == other.name
def jsonify(self):
return {"name": self.name,
"attributes": self.attributes.jsonify()}

View File

@ -31,6 +31,7 @@ from gnocchi import aggregates
from gnocchi import archive_policy
from gnocchi import indexer
from gnocchi import json
from gnocchi import resource_type
from gnocchi import storage
from gnocchi import utils
@ -813,16 +814,16 @@ class ResourceTypeController(rest.RestController):
@pecan.expose('json')
def get(self):
try:
resource_type = pecan.request.indexer.get_resource_type(self._name)
rt = pecan.request.indexer.get_resource_type(self._name)
except indexer.NoSuchResourceType as e:
abort(404, e)
enforce("get resource type", resource_type)
return resource_type
enforce("get resource type", rt)
return rt
@pecan.expose()
def delete(self):
try:
resource_type = pecan.request.indexer.get_resource_type(self._name)
pecan.request.indexer.get_resource_type(self._name)
except indexer.NoSuchResourceType as e:
abort(404, e)
enforce("delete resource type", resource_type)
@ -833,13 +834,6 @@ class ResourceTypeController(rest.RestController):
abort(400, e)
def ResourceTypeSchema(definition):
# FIXME(sileht): Add resource type attributes from the indexer
return voluptuous.Schema({
"name": six.text_type,
})(definition)
class ResourceTypesController(rest.RestController):
@pecan.expose()
@ -848,15 +842,17 @@ class ResourceTypesController(rest.RestController):
@pecan.expose('json')
def post(self):
body = deserialize_and_validate(ResourceTypeSchema)
schema = pecan.request.indexer.get_resource_type_schema()
body = deserialize_and_validate(schema)
rt = schema.resource_type_from_dict(**body)
enforce("create resource type", body)
try:
resource_type = pecan.request.indexer.create_resource_type(**body)
rt = pecan.request.indexer.create_resource_type(rt)
except indexer.ResourceTypeAlreadyExists as e:
abort(409, e)
set_resp_location_hdr("/resource_type/" + resource_type.name)
set_resp_location_hdr("/resource_type/" + rt.name)
pecan.response.status = 201
return resource_type
return rt
@pecan.expose('json')
def get_all(self, **kwargs):
@ -1016,8 +1012,8 @@ def schema_for(resource_type):
# TODO(sileht): Remove this legacy resource schema loading
return RESOURCE_SCHEMA_MANAGER[resource_type].plugin
else:
# TODO(sileht): Load schema from indexer
return GenericSchema
resource_type = pecan.request.indexer.get_resource_type(resource_type)
return ResourceSchema(resource_type.schema)
def ResourceID(value):

View File

@ -23,6 +23,28 @@ tests:
content-type: application/json
status: 403
- name: post resource type bad string
url: /v1/resource_type
method: post
request_headers:
x-roles: admin
content-type: application/json
data:
name: my_custom_resource
attributes:
foo:
type: string
max_length: 32
min_length: 5
noexist: foo
status: 400
response_strings:
# NOTE(sileht): We would prefer to have a better message but voluptuous seems a bit lost when
# an Any have many dict with the same key, here "type"
# - "Invalid input: extra keys not allowed @ data[u'attributes'][u'foo'][u'noexist']"
# - "Invalid input: not a valid value for dictionary value @ data[u'attributes'][u'foo'][u'type']"
- "Invalid input:"
- name: post resource type
url: /v1/resource_type
method: post
@ -31,9 +53,29 @@ tests:
content-type: application/json
data:
name: my_custom_resource
attributes:
name:
type: string
required: true
max_length: 5
min_length: 2
foobar:
type: string
required: false
status: 201
response_json_paths:
$.name: my_custom_resource
$.attributes:
name:
type: string
required: True
max_length: 5
min_length: 2
foobar:
type: string
required: False
max_length: 255
min_length: 0
response_headers:
location: $SCHEME://$NETLOC/v1/resource_type/my_custom_resource
@ -54,6 +96,23 @@ tests:
method: DELETE
status: 403
- name: post invalid resource
url: /v1/resource/my_custom_resource
method: post
request_headers:
x-user-id: 0fbb2314-8461-4b1a-8013-1fc22f6afc9c
x-project-id: f3d41b77-0cc1-4f0b-b94a-1d5be9c0e3ea
content-type: application/json
data:
id: d11edfca-4393-4fda-b94d-b05a3a1b3747
name: toolong!!!
foobar: what
status: 400
response_strings:
# split to not match the u' in py2
- "Invalid input: length of value must be at most 5 for dictionary value @ data["
- "'name']"
- name: post custom resource
url: /v1/resource/my_custom_resource
method: post
@ -63,7 +122,50 @@ tests:
content-type: application/json
data:
id: d11edfca-4393-4fda-b94d-b05a3a1b3747
name: bar
foobar: what
status: 201
response_json_paths:
$.id: d11edfca-4393-4fda-b94d-b05a3a1b3747
$.name: bar
$.foobar: what
- name: patch custom resource
url: /v1/resource/my_custom_resource/d11edfca-4393-4fda-b94d-b05a3a1b3747
method: patch
request_headers:
x-user-id: 0fbb2314-8461-4b1a-8013-1fc22f6afc9c
x-project-id: f3d41b77-0cc1-4f0b-b94a-1d5be9c0e3ea
content-type: application/json
data:
name: foo
status: 200
response_json_paths:
$.id: d11edfca-4393-4fda-b94d-b05a3a1b3747
$.name: foo
$.foobar: what
- name: get resource
url: /v1/resource/my_custom_resource/d11edfca-4393-4fda-b94d-b05a3a1b3747
request_headers:
content-type: application/json
response_json_paths:
$.id: d11edfca-4393-4fda-b94d-b05a3a1b3747
$.name: foo
$.foobar: what
- name: list resource history
url: /v1/resource/my_custom_resource/d11edfca-4393-4fda-b94d-b05a3a1b3747/history?sort=revision_end:asc-nullslast
request_headers:
content-type: application/json
response_json_paths:
$.`len`: 2
$[0].id: d11edfca-4393-4fda-b94d-b05a3a1b3747
$[0].name: bar
$[0].foobar: what
$[1].id: d11edfca-4393-4fda-b94d-b05a3a1b3747
$[1].name: foo
$[1].foobar: what
- name: delete in use resource_type
url: /v1/resource_type/my_custom_resource

View File

@ -974,15 +974,28 @@ class TestIndexerDriver(tests_base.TestCase):
self.assertNotIn(e1, [m.id for m in metrics])
def test_resource_type_crud(self):
mgr = self.index.get_resource_type_schema()
rtype = mgr.resource_type_from_dict("indexer_test", {
"col1": {"type": "string", "required": True,
"min_length": 2, "max_length": 15}
})
# Create
self.index.create_resource_type("indexer_test")
self.index.create_resource_type(rtype)
self.assertRaises(indexer.ResourceTypeAlreadyExists,
self.index.create_resource_type,
"indexer_test")
rtype)
# Get and List
# Get
rtype = self.index.get_resource_type("indexer_test")
self.assertEqual("indexer_test", rtype.name)
self.assertEqual(1, len(rtype.attributes))
self.assertEqual("col1", rtype.attributes[0].name)
self.assertEqual("string", rtype.attributes[0].typename)
self.assertEqual(15, rtype.attributes[0].max_length)
self.assertEqual(2, rtype.attributes[0].min_length)
# List
rtypes = self.index.list_resource_types()
for rtype in rtypes:
if rtype.name == "indexer_test":
@ -994,9 +1007,11 @@ class TestIndexerDriver(tests_base.TestCase):
rid = uuid.uuid4()
self.index.create_resource("indexer_test", rid,
str(uuid.uuid4()),
str(uuid.uuid4()))
str(uuid.uuid4()),
col1="col1_value")
r = self.index.get_resource("indexer_test", rid)
self.assertEqual("indexer_test", r.type)
self.assertEqual("col1_value", r.col1)
# Deletion
self.assertRaises(indexer.ResourceTypeInUse,

View File

@ -85,6 +85,9 @@ data_files =
etc/gnocchi = etc/gnocchi/*
[entry_points]
gnocchi.indexer.sqlalchemy.resource_type_attribute =
string = gnocchi.indexer.sqlalchemy_extension:StringSchema
gnocchi.indexer.resources =
generic = gnocchi.indexer.sqlalchemy_base:Resource
instance = gnocchi.indexer.sqlalchemy_extension:Instance