Merge "Added new rating module PyScripts"

cloudkitty/api/app.py

@@ -48,10 +48,10 @@ auth_opts = [
 api_opts = [
     cfg.StrOpt('host_ip',
                default="0.0.0.0",
-               help="Host serving the API."),
+               help='Host serving the API.'),
     cfg.IntOpt('port',
                default=8888,
-               help="Host port serving the API."),
+               help='Host port serving the API.'),
     cfg.BoolOpt('pecan_debug',
                 default=False,
                 help='Toggle Pecan Debug Middleware.'),

cloudkitty/rating/pyscripts/__init__.py

@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Stéphane Albert
+#
+from oslo_log import log as logging
+
+from cloudkitty import rating
+from cloudkitty.rating.pyscripts.controllers import root as root_api
+from cloudkitty.rating.pyscripts.db import api as pyscripts_db_api
+
+LOG = logging.getLogger(__name__)
+
+
+class PyScripts(rating.RatingProcessorBase):
+    """PyScripts rating module.
+
+    PyScripts is a module made to execute custom made python scripts to create
+    rating policies.
+    """
+
+    module_name = 'pyscripts'
+    description = 'PyScripts rating module.'
+    hot_config = True
+    config_controller = root_api.PyScriptsConfigController
+
+    db_api = pyscripts_db_api.get_instance()
+
+    def __init__(self, tenant_id=None):
+        self._scripts = {}
+        super(PyScripts, self).__init__(tenant_id)
+
+    def load_scripts_in_memory(self):
+        db = pyscripts_db_api.get_instance()
+        scripts_uuid_list = db.list_scripts()
+        # Purge old entries
+        scripts_to_purge = []
+        for script_uuid in self._scripts.keys():
+            if script_uuid not in scripts_uuid_list:
+                scripts_to_purge.append(script_uuid)
+        for script_uuid in scripts_to_purge:
+            del self._scripts[script_uuid]
+        # Load or update script
+        for script_uuid in scripts_uuid_list:
+            script_db = db.get_script(uuid=script_uuid)
+            name = script_db.name
+            checksum = script_db.checksum
+            if name not in self._scripts:
+                self._scripts[script_uuid] = {}
+            script = self._scripts[script_uuid]
+            # NOTE(sheeprine): We're doing this the easy way, we might want to
+            # store the context and call functions in future
+            if script.get(checksum, '') != checksum:
+                code = compile(
+                    script_db.data,
+                    '<PyScripts: {name}>'.format(name=name),
+                    'exec')
+                script.update({
+                    'name': name,
+                    'code': code,
+                    'checksum': checksum})
+
+    def reload_config(self):
+        """Reload the module's configuration.
+
+        """
+        self.load_scripts_in_memory()
+
+    def start_script(self, code, data):
+        context = {'data': data}
+        exec(code, context)
+        return data
+
+    def process(self, data):
+        for script in self._scripts.values():
+            self.start_script(script['code'], data)
+        return data

cloudkitty/rating/pyscripts/controllers/root.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Stéphane Albert
+#
+from cloudkitty import rating
+from cloudkitty.rating.pyscripts.controllers import script as script_api
+
+
+class PyScriptsConfigController(rating.RatingRestControllerBase):
+    """Controller exposing all management sub controllers.
+
+    """
+
+    scripts = script_api.PyScriptsScriptsController()

cloudkitty/rating/pyscripts/controllers/script.py

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Stéphane Albert
+#
+import pecan
+import six
+from wsme import types as wtypes
+import wsmeext.pecan as wsme_pecan
+
+from cloudkitty.api.v1 import types as ck_types
+from cloudkitty import rating
+from cloudkitty.rating.pyscripts.datamodels import script as script_models
+from cloudkitty.rating.pyscripts.db import api as db_api
+
+
+class PyScriptsScriptsController(rating.RatingRestControllerBase):
+    """Controller responsible of scripts management.
+
+    """
+
+    def normalize_data(self, data):
+        """Translate data to binary format if needed.
+
+        :param data: Data to convert to binary type.
+        """
+        if data == wtypes.Unset:
+            return ''
+        if not isinstance(data, six.binary_type):
+            data = data.encode('utf-8')
+        return data
+
+    @wsme_pecan.wsexpose(script_models.ScriptCollection, bool)
+    def get_all(self, no_data=False):
+        """Get the script list
+
+        :param no_data: Set to True to remove script data from output.
+        :return: List of every scripts.
+        """
+        pyscripts = db_api.get_instance()
+        script_list = []
+        script_uuid_list = pyscripts.list_scripts()
+        for script_uuid in script_uuid_list:
+            script_db = pyscripts.get_script(uuid=script_uuid)
+            script = script_db.export_model()
+            if no_data:
+                del script['data']
+            script_list.append(script_models.Script(
+                **script))
+        res = script_models.ScriptCollection(scripts=script_list)
+        return res
+
+    @wsme_pecan.wsexpose(script_models.Script, ck_types.UuidType())
+    def get_one(self, script_id):
+        """Return a script.
+
+        :param script_id: UUID of the script to filter on.
+        """
+        pyscripts = db_api.get_instance()
+        try:
+            script_db = pyscripts.get_script(uuid=script_id)
+            return script_models.Script(**script_db.export_model())
+        except db_api.NoSuchScript as e:
+            pecan.abort(400, six.text_type(e))
+
+    @wsme_pecan.wsexpose(script_models.Script,
+                         body=script_models.Script,
+                         status_code=201)
+    def post(self, script_data):
+        """Create pyscripts script.
+
+        :param script_data: Informations about the script to create.
+        """
+        pyscripts = db_api.get_instance()
+        try:
+            data = self.normalize_data(script_data.data)
+            script_db = pyscripts.create_script(script_data.name, data)
+            pecan.response.location = pecan.request.path_url
+            if pecan.response.location[-1] != '/':
+                pecan.response.location += '/'
+            pecan.response.location += script_db.script_id
+            return script_models.Script(
+                **script_db.export_model())
+        except db_api.ScriptAlreadyExists as e:
+            pecan.abort(409, six.text_type(e))
+
+    @wsme_pecan.wsexpose(script_models.Script,
+                         ck_types.UuidType(),
+                         body=script_models.Script,
+                         status_code=201)
+    def put(self, script_id, script_data):
+        """Update pyscripts script.
+
+        :param script_id: UUID of the script to update.
+        :param script_data: Script data to update.
+        """
+        pyscripts = db_api.get_instance()
+        try:
+            data = self.normalize_data(script_data.data)
+            script_db = pyscripts.update_script(script_id,
+                                                name=script_data.name,
+                                                data=data)
+            pecan.response.location = pecan.request.path_url
+            if pecan.response.location[-1] != '/':
+                pecan.response.location += '/'
+            pecan.response.location += script_db.script_id
+            return script_models.Script(
+                **script_db.export_model())
+        except db_api.NoSuchScript as e:
+            pecan.abort(400, six.text_type(e))
+
+    @wsme_pecan.wsexpose(None, ck_types.UuidType(), status_code=204)
+    def delete(self, script_id):
+        """Delete the script.
+
+        :param script_id: UUID of the script to delete.
+        """
+        pyscripts = db_api.get_instance()
+        try:
+            pyscripts.delete_script(uuid=script_id)
+        except db_api.NoSuchScript as e:
+            pecan.abort(400, six.text_type(e))

cloudkitty/rating/pyscripts/datamodels/script.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Stéphane Albert
+#
+from wsme import types as wtypes
+
+from cloudkitty.api.v1 import types as ck_types
+
+
+class Script(wtypes.Base):
+    """Type describing a script.
+
+    """
+
+    script_id = wtypes.wsattr(ck_types.UuidType(), mandatory=False)
+    """UUID of the script."""
+
+    name = wtypes.wsattr(wtypes.text, mandatory=True)
+    """Name of the script."""
+
+    data = wtypes.wsattr(wtypes.text, mandatory=False)
+    """Data of the script."""
+
+    checksum = wtypes.wsattr(wtypes.text, mandatory=False, readonly=True)
+    """Checksum of the script data."""
+
+    @classmethod
+    def sample(cls):
+        sample = cls(script_id='bc05108d-f515-4984-8077-de319cbf35aa',
+                     name='policy1',
+                     data='return 0',
+                     checksum='da39a3ee5e6b4b0d3255bfef95601890afd80709')
+        return sample
+
+
+class ScriptCollection(wtypes.Base):
+    """Type describing a list of scripts.
+
+    """
+
+    scripts = [Script]
+    """List of scripts."""
+
+    @classmethod
+    def sample(cls):
+        sample = Script.sample()
+        return cls(scripts=[sample])

cloudkitty/rating/pyscripts/db/api.py

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Stéphane Albert
+#
+import abc
+
+from oslo_config import cfg
+from oslo_db import api as db_api
+import six
+
+_BACKEND_MAPPING = {
+    'sqlalchemy': 'cloudkitty.rating.pyscripts.db.sqlalchemy.api'}
+IMPL = db_api.DBAPI.from_config(cfg.CONF,
+                                backend_mapping=_BACKEND_MAPPING,
+                                lazy=True)
+
+
+def get_instance():
+    """Return a DB API instance."""
+    return IMPL
+
+
+class NoSuchScript(Exception):
+    """Raised when the script doesn't exist."""
+
+    def __init__(self, name=None, uuid=None):
+        super(NoSuchScript, self).__init__(
+            "No such script: %s (UUID: %s)" % (name, uuid))
+        self.name = name
+        self.uuid = uuid
+
+
+class ScriptAlreadyExists(Exception):
+    """Raised when the script already exists."""
+
+    def __init__(self, name, uuid):
+        super(ScriptAlreadyExists, self).__init__(
+            "Script %s already exists (UUID: %s)" % (name, uuid))
+        self.name = name
+        self.uuid = uuid
+
+
+@six.add_metaclass(abc.ABCMeta)
+class PyScripts(object):
+    """Base class for pyscripts configuration."""
+
+    @abc.abstractmethod
+    def get_migration(self):
+        """Return a migrate manager.
+
+        """
+
+    @abc.abstractmethod
+    def get_script(self, name=None, uuid=None):
+        """Return a script object.
+
+        :param name: Filter on a script name.
+        :param uuid: The uuid of the script to get.
+        """
+
+    @abc.abstractmethod
+    def list_scripts(self):
+        """Return an UUID list of every scripts available.
+
+        """
+
+    @abc.abstractmethod
+    def create_script(self, name, data):
+        """Create a new script.
+
+        :param name: Name of the script to create.
+        :param data: Content of the python script.
+        """
+
+    @abc.abstractmethod
+    def update_script(self, uuid, **kwargs):
+        """Update a script.
+
+        :param uuid UUID of the script to modify.
+        :param data: Script data.
+        """
+
+    @abc.abstractmethod
+    def delete_script(self, name=None, uuid=None):
+        """Delete a list.
+
+        :param name: Name of the script to delete.
+        :param uuid: UUID of the script to delete.
+        """

cloudkitty/rating/pyscripts/db/sqlalchemy/alembic/env.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Stéphane Albert
+#
+from cloudkitty.common.db.alembic import env  # noqa
+from cloudkitty.rating.pyscripts.db.sqlalchemy import models
+
+target_metadata = models.Base.metadata
+version_table = 'pyscripts_alembic'
+
+
+env.run_migrations_online(target_metadata, version_table)

cloudkitty/rating/pyscripts/db/sqlalchemy/alembic/script.py.mako

@@ -0,0 +1,22 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

cloudkitty/rating/pyscripts/db/sqlalchemy/alembic/versions/4f9efa4601c0_initial_migration.py

@@ -0,0 +1,32 @@
+"""Initial migration.
+
+Revision ID: 4f9efa4601c0
+Revises: None
+Create Date: 2015-07-30 12:46:32.998770
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '4f9efa4601c0'
+down_revision = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.create_table('pyscripts_scripts',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('script_id', sa.String(length=36), nullable=False),
+    sa.Column('name', sa.String(length=255), nullable=False),
+    sa.Column('data', sa.LargeBinary(), nullable=False),
+    sa.Column('checksum', sa.String(length=40), nullable=False),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('name'),
+    sa.UniqueConstraint('script_id'),
+    mysql_charset='utf8',
+    mysql_engine='InnoDB')
+
+
+def downgrade():
+    op.drop_table('pyscripts_scripts')

cloudkitty/rating/pyscripts/db/sqlalchemy/api.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Stéphane Albert
+#
+from oslo_db import exception
+from oslo_db.sqlalchemy import utils
+from oslo_log import log as logging
+from oslo_utils import uuidutils
+import six
+import sqlalchemy
+
+from cloudkitty import db
+from cloudkitty.rating.pyscripts.db import api
+from cloudkitty.rating.pyscripts.db.sqlalchemy import migration
+from cloudkitty.rating.pyscripts.db.sqlalchemy import models
+
+LOG = logging.getLogger(__name__)
+
+
+def get_backend():
+    return PyScripts()
+
+
+class PyScripts(api.PyScripts):
+
+    def get_migration(self):
+        return migration
+
+    def get_script(self, name=None, uuid=None):
+        session = db.get_session()
+        try:
+            q = session.query(models.PyScriptsScript)
+            if name:
+                q = q.filter(
+                    models.PyScriptsScript.name == name)
+            elif uuid:
+                q = q.filter(
+                    models.PyScriptsScript.script_id == uuid)
+            else:
+                raise ValueError('You must specify either name or uuid.')
+            res = q.one()
+            return res
+        except sqlalchemy.orm.exc.NoResultFound:
+            raise api.NoSuchScript(name=name, uuid=uuid)
+
+    def list_scripts(self):
+        session = db.get_session()
+        q = session.query(models.PyScriptsScript)
+        res = q.values(
+            models.PyScriptsScript.script_id)
+        return [uuid[0] for uuid in res]
+
+    def create_script(self, name, data):
+        session = db.get_session()
+        try:
+            with session.begin():
+                script_db = models.PyScriptsScript(name=name)
+                script_db.data = data
+                script_db.script_id = uuidutils.generate_uuid()
+                session.add(script_db)
+            return script_db
+        except exception.DBDuplicateEntry:
+            script_db = self.get_script(name=name)
+            raise api.ScriptAlreadyExists(
+                script_db.name,
+                script_db.script_id)
+
+    def update_script(self, uuid, **kwargs):
+        session = db.get_session()
+        try:
+            with session.begin():
+                q = session.query(models.PyScriptsScript)
+                q = q.filter(
+                    models.PyScriptsScript.script_id == uuid
+                )
+                script_db = q.with_lockmode('update').one()
+                if kwargs:
+                    excluded_cols = ['script_id']
+                    for col in excluded_cols:
+                        if col in kwargs:
+                            kwargs.pop(col)
+                    for attribute, value in six.iteritems(kwargs):
+                        if hasattr(script_db, attribute):
+                            setattr(script_db, attribute, value)
+                        else:
+                            raise ValueError('No such attribute: {}'.format(
+                                attribute))
+                else:
+                    raise ValueError('No attribute to update.')
+                return script_db
+        except sqlalchemy.orm.exc.NoResultFound:
+            raise api.NoSuchScript(uuid=uuid)
+
+    def delete_script(self, name=None, uuid=None):
+        session = db.get_session()
+        q = utils.model_query(
+            models.PyScriptsScript,
+            session)
+        if name:
+            q = q.filter(models.PyScriptsScript.name == name)
+        elif uuid:
+            q = q.filter(models.PyScriptsScript.script_id == uuid)
+        else:
+            raise ValueError('You must specify either name or uuid.')
+        r = q.delete()
+        if not r:
+            raise api.NoSuchScript(uuid=uuid)

cloudkitty/rating/pyscripts/db/sqlalchemy/migration.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Stéphane Albert
+#
+import os
+
+from cloudkitty.common.db.alembic import migration
+
+ALEMBIC_REPO = os.path.join(os.path.dirname(__file__), 'alembic')
+
+
+def upgrade(revision):
+    config = migration.load_alembic_config(ALEMBIC_REPO)
+    return migration.upgrade(config, revision)
+
+
+def downgrade(revision):
+    config = migration.load_alembic_config(ALEMBIC_REPO)
+    return migration.downgrade(config, revision)
+
+
+def version():
+    config = migration.load_alembic_config(ALEMBIC_REPO)
+    return migration.version(config)
+
+
+def revision(message, autogenerate):
+    config = migration.load_alembic_config(ALEMBIC_REPO)
+    return migration.revision(config, message, autogenerate)
+
+
+def stamp(revision):
+    config = migration.load_alembic_config(ALEMBIC_REPO)
+    return migration.stamp(config, revision)

cloudkitty/rating/pyscripts/db/sqlalchemy/models.py

@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Stéphane Albert
+#
+import hashlib
+import zlib
+
+from oslo_db.sqlalchemy import models
+import sqlalchemy
+from sqlalchemy.ext import declarative
+from sqlalchemy.ext import hybrid
+
+Base = declarative.declarative_base()
+
+
+class PyScriptsBase(models.ModelBase):
+    __table_args__ = {'mysql_charset': "utf8",
+                      'mysql_engine': "InnoDB"}
+    fk_to_resolve = {}
+
+    def save(self, session=None):
+        from cloudkitty import db
+
+        if session is None:
+            session = db.get_session()
+
+        super(PyScriptsBase, self).save(session=session)
+
+    def as_dict(self):
+        d = {}
+        for c in self.__table__.columns:
+            if c.name == 'id':
+                continue
+            d[c.name] = self[c.name]
+        return d
+
+    def _recursive_resolve(self, path):
+        obj = self
+        for attr in path.split('.'):
+            if hasattr(obj, attr):
+                obj = getattr(obj, attr)
+            else:
+                return None
+        return obj
+
+    def export_model(self):
+        res = self.as_dict()
+        for fk, mapping in self.fk_to_resolve.items():
+            res[fk] = self._recursive_resolve(mapping)
+        return res
+
+
+class PyScriptsScript(Base, PyScriptsBase):
+    """A PyScripts entry.
+
+    """
+    __tablename__ = 'pyscripts_scripts'
+
+    id = sqlalchemy.Column(sqlalchemy.Integer,
+                           primary_key=True)
+    script_id = sqlalchemy.Column(sqlalchemy.String(36),
+                                  nullable=False,
+                                  unique=True)
+    name = sqlalchemy.Column(
+        sqlalchemy.String(255),
+        nullable=False,
+        unique=True)
+    _data = sqlalchemy.Column('data',
+                              sqlalchemy.LargeBinary(),
+                              nullable=False)
+    _checksum = sqlalchemy.Column('checksum',
+                                  sqlalchemy.String(40),
+                                  nullable=False)
+
+    @hybrid.hybrid_property
+    def data(self):
+        udata = zlib.decompress(self._data)
+        return udata
+
+    @data.setter
+    def data(self, value):
+        sha_check = hashlib.sha1()
+        sha_check.update(value)
+        self._checksum = sha_check.hexdigest()
+        self._data = zlib.compress(value)
+
+    @hybrid.hybrid_property
+    def checksum(self):
+        return self._checksum
+
+    def __repr__(self):
+        return ('<PyScripts Script[{uuid}]: '
+                'name={name}>').format(
+                    uuid=self.script_id,
+                    name=self.name)

cloudkitty/tests/gabbi/rating/pyscripts/fixtures.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Stéphane Albert
+#
+
+from cloudkitty.tests.gabbi.fixtures import *  # noqa
+from cloudkitty.rating.pyscripts.db import api as pyscripts_db
+
+
+class PyScriptsConfigFixture(ConfigFixture):
+    def start_fixture(self):
+        super(PyScriptsConfigFixture, self).start_fixture()
+        self.conn = pyscripts_db.get_instance()
+        migration = self.conn.get_migration()
+        migration.upgrade('head')

cloudkitty/tests/gabbi/rating/pyscripts/gabbits/pyscripts.yaml

@@ -0,0 +1,131 @@
+fixtures:
+    - PyScriptsConfigFixture
+    - UUIDFixture
+
+tests:
+
+  - name: typo of script
+    url: /v1/rating/module_config/pyscripts/script
+    status: 405
+
+  - name: list scripts (empty)
+    url: /v1/rating/module_config/pyscripts/scripts
+    status: 200
+    response_strings:
+      - "[]"
+
+  - name: create policy script
+    url: /v1/rating/module_config/pyscripts/scripts
+    method: POST
+    request_headers:
+        content-type: application/json
+        x-roles: admin
+    data:
+      name: "policy1"
+      data: "a = 0"
+    status: 201
+    response_json_paths:
+      $.script_id: "6c1b8a30-797f-4b7e-ad66-9879b79059fb"
+      $.name: "policy1"
+      $.data: "a = 0"
+      $.checksum: "5ae340c1b3bb81955db1cb593cdc78540082c526"
+    response_headers:
+      location: '$SCHEME://$NETLOC/v1/rating/module_config/pyscripts/scripts/6c1b8a30-797f-4b7e-ad66-9879b79059fb'
+
+  - name: create duplicate policy script
+    url: /v1/rating/module_config/pyscripts/scripts
+    method: POST
+    request_headers:
+      content-type: application/json
+      x-roles: admin
+    data:
+      name: "policy1"
+      data: "a = 0"
+    status: 409
+    response_strings:
+      - "Script policy1 already exists (UUID: 6c1b8a30-797f-4b7e-ad66-9879b79059fb)"
+
+  - name: list scripts
+    url: /v1/rating/module_config/pyscripts/scripts
+    status: 200
+    response_json_paths:
+      $.scripts[0].script_id: "6c1b8a30-797f-4b7e-ad66-9879b79059fb"
+      $.scripts[0].name: "policy1"
+      $.scripts[0].data: "a = 0"
+      $.scripts[0].checksum: "5ae340c1b3bb81955db1cb593cdc78540082c526"
+
+  - name: list scripts excluding data
+    url: /v1/rating/module_config/pyscripts/scripts?no_data=true
+    status: 200
+    response_json_paths:
+      $.scripts[0].script_id: "6c1b8a30-797f-4b7e-ad66-9879b79059fb"
+      $.scripts[0].name: "policy1"
+      $.scripts[0].checksum: "5ae340c1b3bb81955db1cb593cdc78540082c526"
+
+  - name: get script
+    url: /v1/rating/module_config/pyscripts/scripts/6c1b8a30-797f-4b7e-ad66-9879b79059fb
+    status: 200
+    response_json_paths:
+      $.script_id: "6c1b8a30-797f-4b7e-ad66-9879b79059fb"
+      $.name: "policy1"
+      $.data: "a = 0"
+      $.checksum: "5ae340c1b3bb81955db1cb593cdc78540082c526"
+
+  - name: modify script
+    url: /v1/rating/module_config/pyscripts/scripts/6c1b8a30-797f-4b7e-ad66-9879b79059fb
+    method: PUT
+    request_headers:
+        content-type: application/json
+        x-roles: admin
+    data:
+      name: "policy1"
+      data: "a = 1"
+    status: 201
+    response_json_paths:
+      $.script_id: "6c1b8a30-797f-4b7e-ad66-9879b79059fb"
+      $.name: "policy1"
+      $.data: "a = 1"
+      $.checksum: "b88f1ec0c9fe96fde96a6f9dabcbeee661dd7afe"
+
+  - name: modify unknown script
+    url: /v1/rating/module_config/pyscripts/scripts/42
+    method: PUT
+    request_headers:
+        content-type: application/json
+        x-roles: admin
+    data:
+      name: "policy1"
+      data: "a = 1"
+    status: 400
+    response_strings:
+      - "No such script: None (UUID: 42)"
+
+  - name: check updated script
+    url: /v1/rating/module_config/pyscripts/scripts/6c1b8a30-797f-4b7e-ad66-9879b79059fb
+    request_headers:
+        content-type: application/json
+        x-roles: admin
+    status: 200
+    response_json_paths:
+      $.script_id: "6c1b8a30-797f-4b7e-ad66-9879b79059fb"
+      $.name: "policy1"
+      $.data: "a = 1"
+      $.checksum: "b88f1ec0c9fe96fde96a6f9dabcbeee661dd7afe"
+
+  - name: delete script
+    url: /v1/rating/module_config/pyscripts/scripts/6c1b8a30-797f-4b7e-ad66-9879b79059fb
+    method: DELETE
+    status: 204
+
+  - name: get unknown script
+    url: /v1/rating/module_config/pyscripts/scripts/42
+    status: 400
+    response_strings:
+      - "No such script: None (UUID: 42)"
+
+  - name: delete unknown script
+    url: /v1/rating/module_config/pyscripts/scripts/42
+    method: DELETE
+    status: 400
+    response_strings:
+      - "No such script: None (UUID: 42)"

cloudkitty/tests/gabbi/rating/pyscripts/test_gabbi.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Stéphane Albert
+#
+import os
+
+from gabbi import driver
+
+from cloudkitty.tests.gabbi import fixtures
+from cloudkitty.tests.gabbi.rating.pyscripts import fixtures as py_fixtures
+
+TESTS_DIR = 'gabbits'
+
+
+def load_tests(loader, tests, pattern):
+    test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
+    return driver.build_tests(test_dir,
+                              loader,
+                              host=None,
+                              intercept=fixtures.setup_app,
+                              fixture_module=py_fixtures)

cloudkitty/tests/test_pyscripts.py

@@ -0,0 +1,317 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Stéphane Albert
+#
+import copy
+import decimal
+import hashlib
+import zlib
+
+import mock
+from oslo_utils import uuidutils
+import six
+
+from cloudkitty.rating import pyscripts
+from cloudkitty.rating.pyscripts.db import api
+from cloudkitty import tests
+
+FAKE_UUID = '6c1b8a30-797f-4b7e-ad66-9879b79059fb'
+CK_RESOURCES_DATA = [{
+    "period": {
+        "begin": "2014-10-01T00:00:00",
+        "end": "2014-10-01T01:00:00"},
+    "usage": {
+        "compute": [
+            {
+                "desc": {
+                    "availability_zone": "nova",
+                    "flavor": "m1.nano",
+                    "image_id": "f5600101-8fa2-4864-899e-ebcb7ed6b568",
+                    "memory": "64",
+                    "metadata": {
+                        "farm": "prod"},
+                    "name": "prod1",
+                    "project_id": "f266f30b11f246b589fd266f85eeec39",
+                    "user_id": "55b3379b949243009ee96972fbf51ed1",
+                    "vcpus": "1"},
+                "vol": {
+                    "qty": 1,
+                    "unit": "instance"}
+            },
+            {
+                "desc": {
+                    "availability_zone": "nova",
+                    "flavor": "m1.tiny",
+                    "image_id": "a41fba37-2429-4f15-aa00-b5bc4bf557bf",
+                    "memory": "512",
+                    "metadata": {
+                        "farm": "dev"},
+                    "name": "dev1",
+                    "project_id": "f266f30b11f246b589fd266f85eeec39",
+                    "user_id": "55b3379b949243009ee96972fbf51ed1",
+                    "vcpus": "1"},
+                "vol": {
+                    "qty": 2,
+                    "unit": "instance"}},
+            {
+                "desc": {
+                    "availability_zone": "nova",
+                    "flavor": "m1.nano",
+                    "image_id": "a41fba37-2429-4f15-aa00-b5bc4bf557bf",
+                    "memory": "64",
+                    "metadata": {
+                        "farm": "dev"},
+                    "name": "dev2",
+                    "project_id": "f266f30b11f246b589fd266f85eeec39",
+                    "user_id": "55b3379b949243009ee96972fbf51ed1",
+                    "vcpus": "1"},
+                "vol": {
+                    "qty": 1,
+                    "unit": "instance"}}]}}]
+
+TEST_CODE1 = 'a = 1'.encode('utf-8')
+TEST_CODE1_CHECKSUM = hashlib.sha1(TEST_CODE1).hexdigest()
+TEST_CODE2 = 'a = 0'.encode('utf-8')
+TEST_CODE2_CHECKSUM = hashlib.sha1(TEST_CODE2).hexdigest()
+TEST_CODE3 = 'if a == 1: raise Exception()'.encode('utf-8')
+TEST_CODE3_CHECKSUM = hashlib.sha1(TEST_CODE3).hexdigest()
+
+COMPLEX_POLICY1 = """
+import decimal
+
+
+for period in data:
+    for service, resources in period['usage'].items():
+        if service == 'compute':
+            for resource in resources:
+                if resource['desc'].get('flavor') == 'm1.nano':
+                    resource['rating'] = {
+                        'price': decimal.Decimal(1.0)}
+""".encode('utf-8')
+
+
+class PyScriptsRatingTest(tests.TestCase):
+    def setUp(self):
+        super(PyScriptsRatingTest, self).setUp()
+        self._tenant_id = 'f266f30b11f246b589fd266f85eeec39'
+        self._db_api = pyscripts.PyScripts.db_api
+        self._db_api.get_migration().upgrade('head')
+        self._pyscripts = pyscripts.PyScripts(self._tenant_id)
+
+    # Database migrations
+    def test_migration_two_way(self):
+        self._db_api.get_migration().downgrade('base')
+        self._db_api.get_migration().upgrade('head')
+        self._db_api.get_migration().downgrade('base')
+
+    # Scripts tests
+    @mock.patch.object(uuidutils, 'generate_uuid',
+                       return_value=FAKE_UUID)
+    def test_create_script(self, patch_generate_uuid):
+        self._db_api.create_script('policy1', TEST_CODE1)
+        scripts = self._db_api.list_scripts()
+        self.assertEqual([FAKE_UUID], scripts)
+        patch_generate_uuid.assert_called_once_with()
+
+    def test_create_duplicate_script(self):
+        self._db_api.create_script('policy1', TEST_CODE1)
+        self.assertRaises(api.ScriptAlreadyExists,
+                          self._db_api.create_script,
+                          'policy1',
+                          TEST_CODE1)
+
+    def test_get_script_by_uuid(self):
+        expected = self._db_api.create_script('policy1', TEST_CODE1)
+        actual = self._db_api.get_script(uuid=expected.script_id)
+        self.assertEqual(expected.data, actual.data)
+
+    def test_get_script_by_name(self):
+        expected = self._db_api.create_script('policy1', TEST_CODE1)
+        actual = self._db_api.get_script(expected.name)
+        self.assertEqual(expected.data, actual.data)
+
+    def test_get_script_without_parameters(self):
+        self._db_api.create_script('policy1', TEST_CODE1)
+        self.assertRaises(
+            ValueError,
+            self._db_api.get_script)
+
+    def test_delete_script_by_name(self):
+        self._db_api.create_script('policy1', TEST_CODE1)
+        self._db_api.delete_script('policy1')
+        scripts = self._db_api.list_scripts()
+        self.assertEqual([], scripts)
+
+    def test_delete_script_by_uuid(self):
+        script_db = self._db_api.create_script('policy1', TEST_CODE1)
+        self._db_api.delete_script(uuid=script_db.script_id)
+        scripts = self._db_api.list_scripts()
+        self.assertEqual([], scripts)
+
+    def test_delete_script_without_parameters(self):
+        self._db_api.create_script('policy1', TEST_CODE1)
+        self.assertRaises(
+            ValueError,
+            self._db_api.delete_script)
+
+    def test_delete_unknown_script_by_name(self):
+        self.assertRaises(api.NoSuchScript,
+                          self._db_api.delete_script,
+                          'dummy')
+
+    def test_delete_unknown_script_by_uuid(self):
+        self.assertRaises(
+            api.NoSuchScript,
+            self._db_api.delete_script,
+            uuid='6e8de9fc-ee17-4b60-b81a-c9320e994e76')
+
+    def test_update_script(self):
+        script_db = self._db_api.create_script('policy1', TEST_CODE1)
+        self._db_api.update_script(script_db.script_id, data=TEST_CODE2)
+        actual = self._db_api.get_script(uuid=script_db.script_id)
+        self.assertEqual(TEST_CODE2, actual.data)
+
+    def test_update_script_uuid_disabled(self):
+        expected = self._db_api.create_script('policy1', TEST_CODE1)
+        self._db_api.update_script(expected.script_id,
+                                   data=TEST_CODE2,
+                                   script_id='42')
+        actual = self._db_api.get_script(uuid=expected.script_id)
+        self.assertEqual(expected.script_id, actual.script_id)
+
+    def test_update_script_unknown_attribute(self):
+        expected = self._db_api.create_script('policy1', TEST_CODE1)
+        self.assertRaises(
+            ValueError,
+            self._db_api.update_script,
+            expected.script_id,
+            nonexistent=1)
+
+    def test_empty_script_update(self):
+        expected = self._db_api.create_script('policy1', TEST_CODE1)
+        self.assertRaises(
+            ValueError,
+            self._db_api.update_script,
+            expected.script_id)
+
+    # Storage tests
+    def test_compressed_data(self):
+        data = TEST_CODE1
+        self._db_api.create_script('policy1', data)
+        script = self._db_api.get_script('policy1')
+        expected = zlib.compress(data)
+        self.assertEqual(expected, script._data)
+
+    def test_on_the_fly_decompression(self):
+        data = TEST_CODE1
+        self._db_api.create_script('policy1', data)
+        script = self._db_api.get_script('policy1')
+        self.assertEqual(data, script.data)
+
+    def test_script_repr(self):
+        script_db = self._db_api.create_script('policy1', TEST_CODE1)
+        self.assertEqual(
+            '<PyScripts Script[{uuid}]: name={name}>'.format(
+                uuid=script_db.script_id,
+                name=script_db.name),
+            six.text_type(script_db))
+
+    # Checksum tests
+    def test_validate_checksum(self):
+        self._db_api.create_script('policy1', TEST_CODE1)
+        script = self._db_api.get_script('policy1')
+        self.assertEqual(TEST_CODE1_CHECKSUM, script.checksum)
+
+    def test_read_only_checksum(self):
+        self._db_api.create_script('policy1', TEST_CODE1)
+        script = self._db_api.get_script('policy1')
+        self.assertRaises(
+            AttributeError,
+            setattr,
+            script,
+            'checksum',
+            'da39a3ee5e6b4b0d3255bfef95601890afd80709')
+
+    def test_update_checksum(self):
+        self._db_api.create_script('policy1', TEST_CODE1)
+        script = self._db_api.get_script('policy1')
+        script = self._db_api.update_script(script.script_id, data=TEST_CODE2)
+        self.assertEqual(TEST_CODE2_CHECKSUM, script.checksum)
+
+    # Code exec tests
+    def test_load_scripts(self):
+        policy1_db = self._db_api.create_script('policy1', TEST_CODE1)
+        policy2_db = self._db_api.create_script('policy2', TEST_CODE2)
+        self._pyscripts.load_scripts_in_memory()
+        self.assertIn(policy1_db.script_id, self._pyscripts._scripts)
+        self.assertIn(policy2_db.script_id, self._pyscripts._scripts)
+
+    def test_purge_old_scripts(self):
+        policy1_db = self._db_api.create_script('policy1', TEST_CODE1)
+        policy2_db = self._db_api.create_script('policy2', TEST_CODE2)
+        self._pyscripts.reload_config()
+        self.assertIn(policy1_db.script_id, self._pyscripts._scripts)
+        self.assertIn(policy2_db.script_id, self._pyscripts._scripts)
+        self._db_api.delete_script(uuid=policy1_db.script_id)
+        self._pyscripts.reload_config()
+        self.assertNotIn(policy1_db.script_id, self._pyscripts._scripts)
+        self.assertIn(policy2_db.script_id, self._pyscripts._scripts)
+
+    @mock.patch.object(uuidutils, 'generate_uuid',
+                       return_value=FAKE_UUID)
+    def test_valid_script_data_loaded(self, patch_generate_uuid):
+        self._db_api.create_script('policy1', TEST_CODE1)
+        self._pyscripts.load_scripts_in_memory()
+        expected = {
+            FAKE_UUID: {
+                'code': compile(
+                    TEST_CODE1,
+                    '<PyScripts: {name}>'.format(name='policy1'),
+                    'exec'),
+                'checksum': TEST_CODE1_CHECKSUM,
+                'name': 'policy1'
+            }}
+        self.assertEqual(expected, self._pyscripts._scripts)
+        context = {'a': 0}
+        exec(self._pyscripts._scripts[FAKE_UUID]['code'], context)
+        self.assertEqual(1, context['a'])
+
+    def test_update_script_on_checksum_change(self):
+        policy_db = self._db_api.create_script('policy1', TEST_CODE1)
+        self._pyscripts.reload_config()
+        self._db_api.update_script(policy_db.script_id, data=TEST_CODE2)
+        self._pyscripts.reload_config()
+        self.assertEqual(
+            TEST_CODE2_CHECKSUM,
+            self._pyscripts._scripts[policy_db.script_id]['checksum'])
+
+    def test_exec_code_isolation(self):
+        self._db_api.create_script('policy1', TEST_CODE1)
+        self._db_api.create_script('policy2', TEST_CODE3)
+        self._pyscripts.reload_config()
+        self.assertRaises(NameError, self._pyscripts.process, {})
+
+    # Processing
+    def test_process_rating(self):
+        self._db_api.create_script('policy1', COMPLEX_POLICY1)
+        self._pyscripts.reload_config()
+        actual_data = copy.deepcopy(CK_RESOURCES_DATA)
+        expected_data = copy.deepcopy(CK_RESOURCES_DATA)
+        compute_list = expected_data[0]['usage']['compute']
+        compute_list[0]['rating'] = {'price': decimal.Decimal('1')}
+        compute_list[2]['rating'] = {'price': decimal.Decimal('1')}
+        self._pyscripts.process(actual_data)
+        self.assertEqual(expected_data, actual_data)

doc/source/webapi/rating/pyscripts.rst

@@ -0,0 +1,15 @@
+=========================
+PyScripts Module REST API
+=========================
+
+.. rest-controller:: cloudkitty.rating.pyscripts.controllers.root:PyScriptsConfigController
+   :webprefix: /v1/rating/module_config/pyscripts
+
+.. rest-controller:: cloudkitty.rating.pyscripts.controllers.script:PyScriptsScriptsController
+   :webprefix: /v1/rating/module_config/pyscripts/scripts
+
+.. autotype:: cloudkitty.rating.pyscripts.datamodels.script.Script
+   :members:
+
+.. autotype:: cloudkitty.rating.pyscripts.datamodels.script.ScriptCollection
+   :members:

setup.cfg

@@ -50,6 +50,7 @@ cloudkitty.transformers =
 cloudkitty.rating.processors =
     noop = cloudkitty.rating.noop:Noop
     hashmap = cloudkitty.rating.hash:HashMap
+    pyscripts = cloudkitty.rating.pyscripts:PyScripts
 
 cloudkitty.storage.backends =
     sqlalchemy = cloudkitty.storage.sqlalchemy:SQLAlchemyStorage

test-requirements.txt

@@ -4,7 +4,8 @@
 hacking<0.10,>=0.9.2
 coverage>=3.6
 discover
-gabbi>=0.12.0 # Apache-2.0
+gabbi>=1.1.4 # Apache-2.0
+mox3>=0.7.0
 testscenarios>=0.4
 testrepository>=0.0.18
 mock>=1.2