openstack
/
cloudkitty
Code Issues Proposed changes

Added new rating module PyScripts 

PyScripts can execute arbitrary Python code.

Code is stored in the database.
Every piece of code is loaded in memory and executed on the fly.
Code isolation is ensured so that only required data is modified.
Implemented gabbi tests.

Change-Id: Ieb7cd45e53078694603062e425451fa68dc0f791
This commit is contained in:
Stéphane Albert 2015-07-30 12:48:27 +02:00
parent 04a9f0cd43
commit 4482d7daf8
26 changed files with 1296 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cloudkitty/api/app.py
View File

@ -48,10 +48,10 @@ auth_opts = [
api_opts = [
cfg.StrOpt('host_ip',
default="0.0.0.0",
help="Host serving the API."),
help='Host serving the API.'),
cfg.IntOpt('port',
default=8888,
help="Host port serving the API."),
help='Host port serving the API.'),
cfg.BoolOpt('pecan_debug',
default=False,
help='Toggle Pecan Debug Middleware.'),

89
cloudkitty/rating/pyscripts/__init__.py Normal file
View File

@ -0,0 +1,89 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Stéphane Albert
#
from oslo_log import log as logging
from cloudkitty import rating
from cloudkitty.rating.pyscripts.controllers import root as root_api
from cloudkitty.rating.pyscripts.db import api as pyscripts_db_api
LOG = logging.getLogger(__name__)
class PyScripts(rating.RatingProcessorBase):
"""PyScripts rating module.
PyScripts is a module made to execute custom made python scripts to create
rating policies.
"""
module_name = 'pyscripts'
description = 'PyScripts rating module.'
hot_config = True
config_controller = root_api.PyScriptsConfigController
db_api = pyscripts_db_api.get_instance()
def __init__(self, tenant_id=None):
self._scripts = {}
super(PyScripts, self).__init__(tenant_id)
def load_scripts_in_memory(self):
db = pyscripts_db_api.get_instance()
scripts_uuid_list = db.list_scripts()
# Purge old entries
scripts_to_purge = []
for script_uuid in self._scripts.keys():
if script_uuid not in scripts_uuid_list:
scripts_to_purge.append(script_uuid)
for script_uuid in scripts_to_purge:
del self._scripts[script_uuid]
# Load or update script
for script_uuid in scripts_uuid_list:
script_db = db.get_script(uuid=script_uuid)
name = script_db.name
checksum = script_db.checksum
if name not in self._scripts:
self._scripts[script_uuid] = {}
script = self._scripts[script_uuid]
# NOTE(sheeprine): We're doing this the easy way, we might want to
# store the context and call functions in future
if script.get(checksum, '') != checksum:
code = compile(
script_db.data,
'<PyScripts: {name}>'.format(name=name),
'exec')
script.update({
'name': name,
'code': code,
'checksum': checksum})
def reload_config(self):
"""Reload the module's configuration.
"""
self.load_scripts_in_memory()
def start_script(self, code, data):
context = {'data': data}
exec(code, context)
return data
def process(self, data):
for script in self._scripts.values():
self.start_script(script['code'], data)
return data

0
cloudkitty/rating/pyscripts/controllers/__init__.py Normal file
View File

27
cloudkitty/rating/pyscripts/controllers/root.py Normal file
View File

@ -0,0 +1,27 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Stéphane Albert
#
from cloudkitty import rating
from cloudkitty.rating.pyscripts.controllers import script as script_api
class PyScriptsConfigController(rating.RatingRestControllerBase):
"""Controller exposing all management sub controllers.
"""
scripts = script_api.PyScriptsScriptsController()

134
cloudkitty/rating/pyscripts/controllers/script.py Normal file
View File

@ -0,0 +1,134 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Stéphane Albert
#
import pecan
import six
from wsme import types as wtypes
import wsmeext.pecan as wsme_pecan
from cloudkitty.api.v1 import types as ck_types
from cloudkitty import rating
from cloudkitty.rating.pyscripts.datamodels import script as script_models
from cloudkitty.rating.pyscripts.db import api as db_api
class PyScriptsScriptsController(rating.RatingRestControllerBase):
"""Controller responsible of scripts management.
"""
def normalize_data(self, data):
"""Translate data to binary format if needed.
:param data: Data to convert to binary type.
"""
if data == wtypes.Unset:
return ''
if not isinstance(data, six.binary_type):
data = data.encode('utf-8')
return data
@wsme_pecan.wsexpose(script_models.ScriptCollection, bool)
def get_all(self, no_data=False):
"""Get the script list
:param no_data: Set to True to remove script data from output.
:return: List of every scripts.
"""
pyscripts = db_api.get_instance()
script_list = []
script_uuid_list = pyscripts.list_scripts()
for script_uuid in script_uuid_list:
script_db = pyscripts.get_script(uuid=script_uuid)
script = script_db.export_model()
if no_data:
del script['data']
script_list.append(script_models.Script(
**script))
res = script_models.ScriptCollection(scripts=script_list)
return res
@wsme_pecan.wsexpose(script_models.Script, ck_types.UuidType())
def get_one(self, script_id):
"""Return a script.
:param script_id: UUID of the script to filter on.
"""
pyscripts = db_api.get_instance()
try:
script_db = pyscripts.get_script(uuid=script_id)
return script_models.Script(**script_db.export_model())
except db_api.NoSuchScript as e:
pecan.abort(400, six.text_type(e))
@wsme_pecan.wsexpose(script_models.Script,
body=script_models.Script,
status_code=201)
def post(self, script_data):
"""Create pyscripts script.
:param script_data: Informations about the script to create.
"""
pyscripts = db_api.get_instance()
try:
data = self.normalize_data(script_data.data)
script_db = pyscripts.create_script(script_data.name, data)
pecan.response.location = pecan.request.path_url
if pecan.response.location[-1] != '/':
pecan.response.location += '/'
pecan.response.location += script_db.script_id
return script_models.Script(
**script_db.export_model())
except db_api.ScriptAlreadyExists as e:
pecan.abort(409, six.text_type(e))
@wsme_pecan.wsexpose(script_models.Script,
ck_types.UuidType(),
body=script_models.Script,
status_code=201)
def put(self, script_id, script_data):
"""Update pyscripts script.
:param script_id: UUID of the script to update.
:param script_data: Script data to update.
"""
pyscripts = db_api.get_instance()
try:
data = self.normalize_data(script_data.data)
script_db = pyscripts.update_script(script_id,
name=script_data.name,
data=data)
pecan.response.location = pecan.request.path_url
if pecan.response.location[-1] != '/':
pecan.response.location += '/'
pecan.response.location += script_db.script_id
return script_models.Script(
**script_db.export_model())
except db_api.NoSuchScript as e:
pecan.abort(400, six.text_type(e))
@wsme_pecan.wsexpose(None, ck_types.UuidType(), status_code=204)
def delete(self, script_id):
"""Delete the script.
:param script_id: UUID of the script to delete.
"""
pyscripts = db_api.get_instance()
try:
pyscripts.delete_script(uuid=script_id)
except db_api.NoSuchScript as e:
pecan.abort(400, six.text_type(e))

0
cloudkitty/rating/pyscripts/datamodels/__init__.py Normal file
View File

60
cloudkitty/rating/pyscripts/datamodels/script.py Normal file
View File

@ -0,0 +1,60 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Stéphane Albert
#
from wsme import types as wtypes
from cloudkitty.api.v1 import types as ck_types
class Script(wtypes.Base):
"""Type describing a script.
"""
script_id = wtypes.wsattr(ck_types.UuidType(), mandatory=False)
"""UUID of the script."""
name = wtypes.wsattr(wtypes.text, mandatory=True)
"""Name of the script."""
data = wtypes.wsattr(wtypes.text, mandatory=False)
"""Data of the script."""
checksum = wtypes.wsattr(wtypes.text, mandatory=False, readonly=True)
"""Checksum of the script data."""
@classmethod
def sample(cls):
sample = cls(script_id='bc05108d-f515-4984-8077-de319cbf35aa',
name='policy1',
data='return 0',
checksum='da39a3ee5e6b4b0d3255bfef95601890afd80709')
return sample
class ScriptCollection(wtypes.Base):
"""Type describing a list of scripts.
"""
scripts = [Script]
"""List of scripts."""
@classmethod
def sample(cls):
sample = Script.sample()
return cls(scripts=[sample])

0
cloudkitty/rating/pyscripts/db/__init__.py Normal file
View File

102
cloudkitty/rating/pyscripts/db/api.py Normal file
View File

@ -0,0 +1,102 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Stéphane Albert
#
import abc
from oslo_config import cfg
from oslo_db import api as db_api
import six
_BACKEND_MAPPING = {
'sqlalchemy': 'cloudkitty.rating.pyscripts.db.sqlalchemy.api'}
IMPL = db_api.DBAPI.from_config(cfg.CONF,
backend_mapping=_BACKEND_MAPPING,
lazy=True)
def get_instance():
"""Return a DB API instance."""
return IMPL
class NoSuchScript(Exception):
"""Raised when the script doesn't exist."""
def __init__(self, name=None, uuid=None):
super(NoSuchScript, self).__init__(
"No such script: %s (UUID: %s)" % (name, uuid))
self.name = name
self.uuid = uuid
class ScriptAlreadyExists(Exception):
"""Raised when the script already exists."""
def __init__(self, name, uuid):
super(ScriptAlreadyExists, self).__init__(
"Script %s already exists (UUID: %s)" % (name, uuid))
self.name = name
self.uuid = uuid
@six.add_metaclass(abc.ABCMeta)
class PyScripts(object):
"""Base class for pyscripts configuration."""
@abc.abstractmethod
def get_migration(self):
"""Return a migrate manager.
"""
@abc.abstractmethod
def get_script(self, name=None, uuid=None):
"""Return a script object.
:param name: Filter on a script name.
:param uuid: The uuid of the script to get.
"""
@abc.abstractmethod
def list_scripts(self):
"""Return an UUID list of every scripts available.
"""
@abc.abstractmethod
def create_script(self, name, data):
"""Create a new script.
:param name: Name of the script to create.
:param data: Content of the python script.
"""
@abc.abstractmethod
def update_script(self, uuid, **kwargs):
"""Update a script.
:param uuid UUID of the script to modify.
:param data: Script data.
"""
@abc.abstractmethod
def delete_script(self, name=None, uuid=None):
"""Delete a list.
:param name: Name of the script to delete.
:param uuid: UUID of the script to delete.
"""

0
cloudkitty/rating/pyscripts/db/sqlalchemy/__init__.py Normal file
View File

0
cloudkitty/rating/pyscripts/db/sqlalchemy/alembic/__init__.py Normal file
View File

25
cloudkitty/rating/pyscripts/db/sqlalchemy/alembic/env.py Normal file
View File

@ -0,0 +1,25 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Stéphane Albert
#
from cloudkitty.common.db.alembic import env # noqa
from cloudkitty.rating.pyscripts.db.sqlalchemy import models
target_metadata = models.Base.metadata
version_table = 'pyscripts_alembic'
env.run_migrations_online(target_metadata, version_table)

22
cloudkitty/rating/pyscripts/db/sqlalchemy/alembic/script.py.mako Normal file
View File

@ -0,0 +1,22 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision}
Create Date: ${create_date}
"""
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

32
cloudkitty/rating/pyscripts/db/sqlalchemy/alembic/versions/4f9efa4601c0_initial_migration.py Normal file
View File

@ -0,0 +1,32 @@
"""Initial migration.
Revision ID: 4f9efa4601c0
Revises: None
Create Date: 2015-07-30 12:46:32.998770
"""
# revision identifiers, used by Alembic.
revision = '4f9efa4601c0'
down_revision = None
from alembic import op
import sqlalchemy as sa
def upgrade():
op.create_table('pyscripts_scripts',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('script_id', sa.String(length=36), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.Column('data', sa.LargeBinary(), nullable=False),
sa.Column('checksum', sa.String(length=40), nullable=False),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name'),
sa.UniqueConstraint('script_id'),
mysql_charset='utf8',
mysql_engine='InnoDB')
def downgrade():
op.drop_table('pyscripts_scripts')

120
cloudkitty/rating/pyscripts/db/sqlalchemy/api.py Normal file
View File

@ -0,0 +1,120 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Stéphane Albert
#
from oslo_db import exception
from oslo_db.sqlalchemy import utils
from oslo_log import log as logging
from oslo_utils import uuidutils
import six
import sqlalchemy
from cloudkitty import db
from cloudkitty.rating.pyscripts.db import api
from cloudkitty.rating.pyscripts.db.sqlalchemy import migration
from cloudkitty.rating.pyscripts.db.sqlalchemy import models
LOG = logging.getLogger(__name__)
def get_backend():
return PyScripts()
class PyScripts(api.PyScripts):
def get_migration(self):
return migration
def get_script(self, name=None, uuid=None):
session = db.get_session()
try:
q = session.query(models.PyScriptsScript)
if name:
q = q.filter(
models.PyScriptsScript.name == name)
elif uuid:
q = q.filter(
models.PyScriptsScript.script_id == uuid)
else:
raise ValueError('You must specify either name or uuid.')
res = q.one()
return res
except sqlalchemy.orm.exc.NoResultFound:
raise api.NoSuchScript(name=name, uuid=uuid)
def list_scripts(self):
session = db.get_session()
q = session.query(models.PyScriptsScript)
res = q.values(
models.PyScriptsScript.script_id)
return [uuid[0] for uuid in res]
def create_script(self, name, data):
session = db.get_session()
try:
with session.begin():
script_db = models.PyScriptsScript(name=name)
script_db.data = data
script_db.script_id = uuidutils.generate_uuid()
session.add(script_db)
return script_db
except exception.DBDuplicateEntry:
script_db = self.get_script(name=name)
raise api.ScriptAlreadyExists(
script_db.name,
script_db.script_id)
def update_script(self, uuid, **kwargs):
session = db.get_session()
try:
with session.begin():
q = session.query(models.PyScriptsScript)
q = q.filter(
models.PyScriptsScript.script_id == uuid
)
script_db = q.with_lockmode('update').one()
if kwargs:
excluded_cols = ['script_id']
for col in excluded_cols:
if col in kwargs:
kwargs.pop(col)
for attribute, value in six.iteritems(kwargs):
if hasattr(script_db, attribute):
setattr(script_db, attribute, value)
else:
raise ValueError('No such attribute: {}'.format(
attribute))
else:
raise ValueError('No attribute to update.')
return script_db
except sqlalchemy.orm.exc.NoResultFound:
raise api.NoSuchScript(uuid=uuid)
def delete_script(self, name=None, uuid=None):
session = db.get_session()
q = utils.model_query(
models.PyScriptsScript,
session)
if name:
q = q.filter(models.PyScriptsScript.name == name)
elif uuid:
q = q.filter(models.PyScriptsScript.script_id == uuid)
else:
raise ValueError('You must specify either name or uuid.')
r = q.delete()
if not r:
raise api.NoSuchScript(uuid=uuid)

47
cloudkitty/rating/pyscripts/db/sqlalchemy/migration.py Normal file
View File

@ -0,0 +1,47 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Stéphane Albert
#
import os
from cloudkitty.common.db.alembic import migration
ALEMBIC_REPO = os.path.join(os.path.dirname(__file__), 'alembic')
def upgrade(revision):
config = migration.load_alembic_config(ALEMBIC_REPO)
return migration.upgrade(config, revision)
def downgrade(revision):
config = migration.load_alembic_config(ALEMBIC_REPO)
return migration.downgrade(config, revision)
def version():
config = migration.load_alembic_config(ALEMBIC_REPO)
return migration.version(config)
def revision(message, autogenerate):
config = migration.load_alembic_config(ALEMBIC_REPO)
return migration.revision(config, message, autogenerate)
def stamp(revision):
config = migration.load_alembic_config(ALEMBIC_REPO)
return migration.stamp(config, revision)

108
cloudkitty/rating/pyscripts/db/sqlalchemy/models.py Normal file
View File

@ -0,0 +1,108 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Stéphane Albert
#
import hashlib
import zlib
from oslo_db.sqlalchemy import models
import sqlalchemy
from sqlalchemy.ext import declarative
from sqlalchemy.ext import hybrid
Base = declarative.declarative_base()
class PyScriptsBase(models.ModelBase):
__table_args__ = {'mysql_charset': "utf8",
'mysql_engine': "InnoDB"}
fk_to_resolve = {}
def save(self, session=None):
from cloudkitty import db
if session is None:
session = db.get_session()
super(PyScriptsBase, self).save(session=session)
def as_dict(self):
d = {}
for c in self.__table__.columns:
if c.name == 'id':
continue
d[c.name] = self[c.name]
return d
def _recursive_resolve(self, path):
obj = self
for attr in path.split('.'):
if hasattr(obj, attr):
obj = getattr(obj, attr)
else:
return None
return obj
def export_model(self):
res = self.as_dict()
for fk, mapping in self.fk_to_resolve.items():
res[fk] = self._recursive_resolve(mapping)
return res
class PyScriptsScript(Base, PyScriptsBase):
"""A PyScripts entry.
"""
__tablename__ = 'pyscripts_scripts'
id = sqlalchemy.Column(sqlalchemy.Integer,
primary_key=True)
script_id = sqlalchemy.Column(sqlalchemy.String(36),
nullable=False,
unique=True)
name = sqlalchemy.Column(
sqlalchemy.String(255),
nullable=False,
unique=True)
_data = sqlalchemy.Column('data',
sqlalchemy.LargeBinary(),
nullable=False)
_checksum = sqlalchemy.Column('checksum',
sqlalchemy.String(40),
nullable=False)
@hybrid.hybrid_property
def data(self):
udata = zlib.decompress(self._data)
return udata
@data.setter
def data(self, value):
sha_check = hashlib.sha1()
sha_check.update(value)
self._checksum = sha_check.hexdigest()
self._data = zlib.compress(value)
@hybrid.hybrid_property
def checksum(self):
return self._checksum
def __repr__(self):
return ('<PyScripts Script[{uuid}]: '
'name={name}>').format(
uuid=self.script_id,
name=self.name)

0
cloudkitty/tests/gabbi/rating/__init__.py Normal file
View File

0
cloudkitty/tests/gabbi/rating/pyscripts/__init__.py Normal file
View File

28
cloudkitty/tests/gabbi/rating/pyscripts/fixtures.py Normal file
View File

@ -0,0 +1,28 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Stéphane Albert
#
from cloudkitty.tests.gabbi.fixtures import * # noqa
from cloudkitty.rating.pyscripts.db import api as pyscripts_db
class PyScriptsConfigFixture(ConfigFixture):
def start_fixture(self):
super(PyScriptsConfigFixture, self).start_fixture()
self.conn = pyscripts_db.get_instance()
migration = self.conn.get_migration()
migration.upgrade('head')

131
cloudkitty/tests/gabbi/rating/pyscripts/gabbits/pyscripts.yaml Normal file
View File

@ -0,0 +1,131 @@
fixtures:
- PyScriptsConfigFixture
- UUIDFixture
tests:
- name: typo of script
url: /v1/rating/module_config/pyscripts/script
status: 405
- name: list scripts (empty)
url: /v1/rating/module_config/pyscripts/scripts
status: 200
response_strings:
- "[]"
- name: create policy script
url: /v1/rating/module_config/pyscripts/scripts
method: POST
request_headers:
content-type: application/json
x-roles: admin
data:
name: "policy1"
data: "a = 0"
status: 201
response_json_paths:
$.script_id: "6c1b8a30-797f-4b7e-ad66-9879b79059fb"
$.name: "policy1"
$.data: "a = 0"
$.checksum: "5ae340c1b3bb81955db1cb593cdc78540082c526"
response_headers:
location: '$SCHEME://$NETLOC/v1/rating/module_config/pyscripts/scripts/6c1b8a30-797f-4b7e-ad66-9879b79059fb'
- name: create duplicate policy script
url: /v1/rating/module_config/pyscripts/scripts
method: POST
request_headers:
content-type: application/json
x-roles: admin
data:
name: "policy1"
data: "a = 0"
status: 409
response_strings:
- "Script policy1 already exists (UUID: 6c1b8a30-797f-4b7e-ad66-9879b79059fb)"
- name: list scripts
url: /v1/rating/module_config/pyscripts/scripts
status: 200
response_json_paths:
$.scripts[0].script_id: "6c1b8a30-797f-4b7e-ad66-9879b79059fb"
$.scripts[0].name: "policy1"
$.scripts[0].data: "a = 0"
$.scripts[0].checksum: "5ae340c1b3bb81955db1cb593cdc78540082c526"
- name: list scripts excluding data
url: /v1/rating/module_config/pyscripts/scripts?no_data=true
status: 200
response_json_paths:
$.scripts[0].script_id: "6c1b8a30-797f-4b7e-ad66-9879b79059fb"
$.scripts[0].name: "policy1"
$.scripts[0].checksum: "5ae340c1b3bb81955db1cb593cdc78540082c526"
- name: get script
url: /v1/rating/module_config/pyscripts/scripts/6c1b8a30-797f-4b7e-ad66-9879b79059fb
status: 200
response_json_paths:
$.script_id: "6c1b8a30-797f-4b7e-ad66-9879b79059fb"
$.name: "policy1"
$.data: "a = 0"
$.checksum: "5ae340c1b3bb81955db1cb593cdc78540082c526"
- name: modify script
url: /v1/rating/module_config/pyscripts/scripts/6c1b8a30-797f-4b7e-ad66-9879b79059fb
method: PUT
request_headers:
content-type: application/json
x-roles: admin
data:
name: "policy1"
data: "a = 1"
status: 201
response_json_paths:
$.script_id: "6c1b8a30-797f-4b7e-ad66-9879b79059fb"
$.name: "policy1"
$.data: "a = 1"
$.checksum: "b88f1ec0c9fe96fde96a6f9dabcbeee661dd7afe"
- name: modify unknown script
url: /v1/rating/module_config/pyscripts/scripts/42
method: PUT
request_headers:
content-type: application/json
x-roles: admin
data:
name: "policy1"
data: "a = 1"
status: 400
response_strings:
- "No such script: None (UUID: 42)"
- name: check updated script
url: /v1/rating/module_config/pyscripts/scripts/6c1b8a30-797f-4b7e-ad66-9879b79059fb
request_headers:
content-type: application/json
x-roles: admin
status: 200
response_json_paths:
$.script_id: "6c1b8a30-797f-4b7e-ad66-9879b79059fb"
$.name: "policy1"
$.data: "a = 1"
$.checksum: "b88f1ec0c9fe96fde96a6f9dabcbeee661dd7afe"
- name: delete script
url: /v1/rating/module_config/pyscripts/scripts/6c1b8a30-797f-4b7e-ad66-9879b79059fb
method: DELETE
status: 204
- name: get unknown script
url: /v1/rating/module_config/pyscripts/scripts/42
status: 400
response_strings:
- "No such script: None (UUID: 42)"
- name: delete unknown script
url: /v1/rating/module_config/pyscripts/scripts/42
method: DELETE
status: 400
response_strings:
- "No such script: None (UUID: 42)"

34
cloudkitty/tests/gabbi/rating/pyscripts/test_gabbi.py Normal file
View File

@ -0,0 +1,34 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Stéphane Albert
#
import os
from gabbi import driver
from cloudkitty.tests.gabbi import fixtures
from cloudkitty.tests.gabbi.rating.pyscripts import fixtures as py_fixtures
TESTS_DIR = 'gabbits'
def load_tests(loader, tests, pattern):
test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
return driver.build_tests(test_dir,
loader,
host=None,
intercept=fixtures.setup_app,
fixture_module=py_fixtures)

317
cloudkitty/tests/test_pyscripts.py Normal file
View File

@ -0,0 +1,317 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Stéphane Albert
#
import copy
import decimal
import hashlib
import zlib
import mock
from oslo_utils import uuidutils
import six
from cloudkitty.rating import pyscripts
from cloudkitty.rating.pyscripts.db import api
from cloudkitty import tests
FAKE_UUID = '6c1b8a30-797f-4b7e-ad66-9879b79059fb'
CK_RESOURCES_DATA = [{
"period": {
"begin": "2014-10-01T00:00:00",
"end": "2014-10-01T01:00:00"},
"usage": {
"compute": [
{
"desc": {
"availability_zone": "nova",
"flavor": "m1.nano",
"image_id": "f5600101-8fa2-4864-899e-ebcb7ed6b568",
"memory": "64",
"metadata": {
"farm": "prod"},
"name": "prod1",
"project_id": "f266f30b11f246b589fd266f85eeec39",
"user_id": "55b3379b949243009ee96972fbf51ed1",
"vcpus": "1"},
"vol": {
"qty": 1,
"unit": "instance"}
},
{
"desc": {
"availability_zone": "nova",
"flavor": "m1.tiny",
"image_id": "a41fba37-2429-4f15-aa00-b5bc4bf557bf",
"memory": "512",
"metadata": {
"farm": "dev"},
"name": "dev1",
"project_id": "f266f30b11f246b589fd266f85eeec39",
"user_id": "55b3379b949243009ee96972fbf51ed1",
"vcpus": "1"},
"vol": {
"qty": 2,
"unit": "instance"}},
{
"desc": {
"availability_zone": "nova",
"flavor": "m1.nano",
"image_id": "a41fba37-2429-4f15-aa00-b5bc4bf557bf",
"memory": "64",
"metadata": {
"farm": "dev"},
"name": "dev2",
"project_id": "f266f30b11f246b589fd266f85eeec39",
"user_id": "55b3379b949243009ee96972fbf51ed1",
"vcpus": "1"},
"vol": {
"qty": 1,
"unit": "instance"}}]}}]
TEST_CODE1 = 'a = 1'.encode('utf-8')
TEST_CODE1_CHECKSUM = hashlib.sha1(TEST_CODE1).hexdigest()
TEST_CODE2 = 'a = 0'.encode('utf-8')
TEST_CODE2_CHECKSUM = hashlib.sha1(TEST_CODE2).hexdigest()
TEST_CODE3 = 'if a == 1: raise Exception()'.encode('utf-8')
TEST_CODE3_CHECKSUM = hashlib.sha1(TEST_CODE3).hexdigest()
COMPLEX_POLICY1 = """
import decimal
for period in data:
for service, resources in period['usage'].items():
if service == 'compute':
for resource in resources:
if resource['desc'].get('flavor') == 'm1.nano':
resource['rating'] = {
'price': decimal.Decimal(1.0)}
""".encode('utf-8')
class PyScriptsRatingTest(tests.TestCase):
def setUp(self):
super(PyScriptsRatingTest, self).setUp()
self._tenant_id = 'f266f30b11f246b589fd266f85eeec39'
self._db_api = pyscripts.PyScripts.db_api
self._db_api.get_migration().upgrade('head')
self._pyscripts = pyscripts.PyScripts(self._tenant_id)
# Database migrations
def test_migration_two_way(self):
self._db_api.get_migration().downgrade('base')
self._db_api.get_migration().upgrade('head')
self._db_api.get_migration().downgrade('base')
# Scripts tests
@mock.patch.object(uuidutils, 'generate_uuid',
return_value=FAKE_UUID)
def test_create_script(self, patch_generate_uuid):
self._db_api.create_script('policy1', TEST_CODE1)
scripts = self._db_api.list_scripts()
self.assertEqual([FAKE_UUID], scripts)
patch_generate_uuid.assert_called_once_with()
def test_create_duplicate_script(self):
self._db_api.create_script('policy1', TEST_CODE1)
self.assertRaises(api.ScriptAlreadyExists,
self._db_api.create_script,
'policy1',
TEST_CODE1)
def test_get_script_by_uuid(self):
expected = self._db_api.create_script('policy1', TEST_CODE1)
actual = self._db_api.get_script(uuid=expected.script_id)
self.assertEqual(expected.data, actual.data)
def test_get_script_by_name(self):
expected = self._db_api.create_script('policy1', TEST_CODE1)
actual = self._db_api.get_script(expected.name)
self.assertEqual(expected.data, actual.data)
def test_get_script_without_parameters(self):
self._db_api.create_script('policy1', TEST_CODE1)
self.assertRaises(
ValueError,
self._db_api.get_script)
def test_delete_script_by_name(self):
self._db_api.create_script('policy1', TEST_CODE1)
self._db_api.delete_script('policy1')
scripts = self._db_api.list_scripts()
self.assertEqual([], scripts)
def test_delete_script_by_uuid(self):
script_db = self._db_api.create_script('policy1', TEST_CODE1)
self._db_api.delete_script(uuid=script_db.script_id)
scripts = self._db_api.list_scripts()
self.assertEqual([], scripts)
def test_delete_script_without_parameters(self):
self._db_api.create_script('policy1', TEST_CODE1)
self.assertRaises(
ValueError,
self._db_api.delete_script)
def test_delete_unknown_script_by_name(self):
self.assertRaises(api.NoSuchScript,
self._db_api.delete_script,
'dummy')
def test_delete_unknown_script_by_uuid(self):
self.assertRaises(
api.NoSuchScript,
self._db_api.delete_script,
uuid='6e8de9fc-ee17-4b60-b81a-c9320e994e76')
def test_update_script(self):
script_db = self._db_api.create_script('policy1', TEST_CODE1)
self._db_api.update_script(script_db.script_id, data=TEST_CODE2)
actual = self._db_api.get_script(uuid=script_db.script_id)
self.assertEqual(TEST_CODE2, actual.data)
def test_update_script_uuid_disabled(self):
expected = self._db_api.create_script('policy1', TEST_CODE1)
self._db_api.update_script(expected.script_id,
data=TEST_CODE2,
script_id='42')
actual = self._db_api.get_script(uuid=expected.script_id)
self.assertEqual(expected.script_id, actual.script_id)
def test_update_script_unknown_attribute(self):
expected = self._db_api.create_script('policy1', TEST_CODE1)
self.assertRaises(
ValueError,
self._db_api.update_script,
expected.script_id,
nonexistent=1)
def test_empty_script_update(self):
expected = self._db_api.create_script('policy1', TEST_CODE1)
self.assertRaises(
ValueError,
self._db_api.update_script,
expected.script_id)
# Storage tests
def test_compressed_data(self):
data = TEST_CODE1
self._db_api.create_script('policy1', data)
script = self._db_api.get_script('policy1')
expected = zlib.compress(data)
self.assertEqual(expected, script._data)
def test_on_the_fly_decompression(self):
data = TEST_CODE1
self._db_api.create_script('policy1', data)
script = self._db_api.get_script('policy1')
self.assertEqual(data, script.data)
def test_script_repr(self):
script_db = self._db_api.create_script('policy1', TEST_CODE1)
self.assertEqual(
'<PyScripts Script[{uuid}]: name={name}>'.format(
uuid=script_db.script_id,
name=script_db.name),
six.text_type(script_db))
# Checksum tests
def test_validate_checksum(self):
self._db_api.create_script('policy1', TEST_CODE1)
script = self._db_api.get_script('policy1')
self.assertEqual(TEST_CODE1_CHECKSUM, script.checksum)
def test_read_only_checksum(self):
self._db_api.create_script('policy1', TEST_CODE1)
script = self._db_api.get_script('policy1')
self.assertRaises(
AttributeError,
setattr,
script,
'checksum',
'da39a3ee5e6b4b0d3255bfef95601890afd80709')
def test_update_checksum(self):
self._db_api.create_script('policy1', TEST_CODE1)
script = self._db_api.get_script('policy1')
script = self._db_api.update_script(script.script_id, data=TEST_CODE2)
self.assertEqual(TEST_CODE2_CHECKSUM, script.checksum)
# Code exec tests
def test_load_scripts(self):
policy1_db = self._db_api.create_script('policy1', TEST_CODE1)
policy2_db = self._db_api.create_script('policy2', TEST_CODE2)
self._pyscripts.load_scripts_in_memory()
self.assertIn(policy1_db.script_id, self._pyscripts._scripts)
self.assertIn(policy2_db.script_id, self._pyscripts._scripts)
def test_purge_old_scripts(self):
policy1_db = self._db_api.create_script('policy1', TEST_CODE1)
policy2_db = self._db_api.create_script('policy2', TEST_CODE2)
self._pyscripts.reload_config()
self.assertIn(policy1_db.script_id, self._pyscripts._scripts)
self.assertIn(policy2_db.script_id, self._pyscripts._scripts)
self._db_api.delete_script(uuid=policy1_db.script_id)
self._pyscripts.reload_config()
self.assertNotIn(policy1_db.script_id, self._pyscripts._scripts)
self.assertIn(policy2_db.script_id, self._pyscripts._scripts)
@mock.patch.object(uuidutils, 'generate_uuid',
return_value=FAKE_UUID)
def test_valid_script_data_loaded(self, patch_generate_uuid):
self._db_api.create_script('policy1', TEST_CODE1)
self._pyscripts.load_scripts_in_memory()
expected = {
FAKE_UUID: {
'code': compile(
TEST_CODE1,
'<PyScripts: {name}>'.format(name='policy1'),
'exec'),
'checksum': TEST_CODE1_CHECKSUM,
'name': 'policy1'
}}
self.assertEqual(expected, self._pyscripts._scripts)
context = {'a': 0}
exec(self._pyscripts._scripts[FAKE_UUID]['code'], context)
self.assertEqual(1, context['a'])
def test_update_script_on_checksum_change(self):
policy_db = self._db_api.create_script('policy1', TEST_CODE1)
self._pyscripts.reload_config()
self._db_api.update_script(policy_db.script_id, data=TEST_CODE2)
self._pyscripts.reload_config()
self.assertEqual(
TEST_CODE2_CHECKSUM,
self._pyscripts._scripts[policy_db.script_id]['checksum'])
def test_exec_code_isolation(self):
self._db_api.create_script('policy1', TEST_CODE1)
self._db_api.create_script('policy2', TEST_CODE3)
self._pyscripts.reload_config()
self.assertRaises(NameError, self._pyscripts.process, {})
# Processing
def test_process_rating(self):
self._db_api.create_script('policy1', COMPLEX_POLICY1)
self._pyscripts.reload_config()
actual_data = copy.deepcopy(CK_RESOURCES_DATA)
expected_data = copy.deepcopy(CK_RESOURCES_DATA)
compute_list = expected_data[0]['usage']['compute']
compute_list[0]['rating'] = {'price': decimal.Decimal('1')}
compute_list[2]['rating'] = {'price': decimal.Decimal('1')}
self._pyscripts.process(actual_data)
self.assertEqual(expected_data, actual_data)

15
doc/source/webapi/rating/pyscripts.rst Normal file
View File

@ -0,0 +1,15 @@
=========================
PyScripts Module REST API
=========================
.. rest-controller:: cloudkitty.rating.pyscripts.controllers.root:PyScriptsConfigController
:webprefix: /v1/rating/module_config/pyscripts
.. rest-controller:: cloudkitty.rating.pyscripts.controllers.script:PyScriptsScriptsController
:webprefix: /v1/rating/module_config/pyscripts/scripts
.. autotype:: cloudkitty.rating.pyscripts.datamodels.script.Script
:members:
.. autotype:: cloudkitty.rating.pyscripts.datamodels.script.ScriptCollection
:members:

1
setup.cfg
View File

@ -50,6 +50,7 @@ cloudkitty.transformers =
cloudkitty.rating.processors =
noop = cloudkitty.rating.noop:Noop
hashmap = cloudkitty.rating.hash:HashMap
pyscripts = cloudkitty.rating.pyscripts:PyScripts
cloudkitty.storage.backends =
sqlalchemy = cloudkitty.storage.sqlalchemy:SQLAlchemyStorage

3
test-requirements.txt
View File

@ -4,7 +4,8 @@
hacking<0.10,>=0.9.2
coverage>=3.6
discover
gabbi>=0.12.0 # Apache-2.0
gabbi>=1.1.4 # Apache-2.0
mox3>=0.7.0
testscenarios>=0.4
testrepository>=0.0.18
mock>=1.2