openstack/cloudkitty
Code Issues Proposed changes

Fix PyScripts processing

Browse Source 
The PyScript process in CloudKitty has been broken for a very long
time. This patch introduces changes required to make it work again.

Change-Id: I143ee6aa4352903921d2ab7b8d8468aedbdd6911
This commit is contained in:
Rafael Weingärtner 2022-11-15 15:27:18 -03:00
parent 0c1eabc364
commit ee99f7ef0d
6 changed files with 331 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cloudkitty/dataframe.py
View File

@ -37,8 +37,8 @@ DATAPOINT_SCHEMA = voluptuous.Schema({
voluptuous.Required('price', default=0): voluptuous.Required('price', default=0):
voluptuous.Coerce(str), voluptuous.Coerce(str),
}, },
voluptuous.Required('groupby'): vutils.DictTypeValidator(str, str), voluptuous.Required('groupby'): voluptuous.Coerce(dict),
voluptuous.Required('metadata'): vutils.DictTypeValidator(str, str), voluptuous.Required('metadata'): voluptuous.Coerce(dict),
}) })

47
cloudkitty/rating/pyscripts/__init__.py
View File

@ -13,10 +13,15 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
# #
from cloudkitty import dataframe
from cloudkitty import rating from cloudkitty import rating
from cloudkitty.rating.pyscripts.controllers import root as root_api from cloudkitty.rating.pyscripts.controllers import root as root_api
from cloudkitty.rating.pyscripts.db import api as pyscripts_db_api from cloudkitty.rating.pyscripts.db import api as pyscripts_db_api
from oslo_log import log as logging
LOG = logging.getLogger(__name__)
class PyScripts(rating.RatingProcessorBase): class PyScripts(rating.RatingProcessorBase):
"""PyScripts rating module. """PyScripts rating module.
@ -33,20 +38,17 @@ class PyScripts(rating.RatingProcessorBase):
db_api = pyscripts_db_api.get_instance() db_api = pyscripts_db_api.get_instance()
def __init__(self, tenant_id=None): def __init__(self, tenant_id=None):
# current scripts loaded to memory
self._scripts = {} self._scripts = {}
self.load_scripts_in_memory() self.load_scripts_in_memory()
super(PyScripts, self).__init__(tenant_id) super(PyScripts, self).__init__(tenant_id)
def load_scripts_in_memory(self): def load_scripts_in_memory(self):
db = pyscripts_db_api.get_instance() db = pyscripts_db_api.get_instance()
scripts_uuid_list = db.list_scripts() scripts_uuid_list = db.list_scripts()
# Purge old entries self.purge_removed_scripts(scripts_uuid_list)
scripts_to_purge = []
for script_uuid in self._scripts.keys():
if script_uuid not in scripts_uuid_list:
scripts_to_purge.append(script_uuid)
for script_uuid in scripts_to_purge:
del self._scripts[script_uuid]
# Load or update script # Load or update script
for script_uuid in scripts_uuid_list: for script_uuid in scripts_uuid_list:
script_db = db.get_script(uuid=script_uuid) script_db = db.get_script(uuid=script_uuid)
@ -67,11 +69,31 @@ class PyScripts(rating.RatingProcessorBase):
'code': code, 'code': code,
'checksum': checksum}) 'checksum': checksum})
def purge_removed_scripts(self, scripts_uuid_list):
scripts_to_purge = self.get_all_script_to_remove(scripts_uuid_list)
self.remove_purged_scripts(scripts_to_purge)
def get_all_script_to_remove(self, new_scripts_uuid_list):
scripts_to_purge = []
for script_uuid in self._scripts.keys():
if script_uuid not in new_scripts_uuid_list:
scripts_to_purge.append(script_uuid)
return scripts_to_purge
def remove_purged_scripts(self, scripts_to_purge):
for script_uuid in scripts_to_purge:
LOG.info("Removing script [%s] from the script list to execute.",
self._scripts[script_uuid])
del self._scripts[script_uuid]
def reload_config(self): def reload_config(self):
"""Reload the module's configuration. """Reload the module's configuration.
""" """
LOG.debug("Executing the reload of configurations.")
self.load_scripts_in_memory() self.load_scripts_in_memory()
LOG.debug("Configurations reloaded.")
def start_script(self, code, data): def start_script(self, code, data):
context = {'data': data} context = {'data': data}
@ -80,5 +102,14 @@ class PyScripts(rating.RatingProcessorBase):
def process(self, data): def process(self, data):
for script in self._scripts.values(): for script in self._scripts.values():
data = self.start_script(script['code'], data) data_dict = data.as_dict(mutable=True)
LOG.debug("Executing pyscript [%s] with data [%s].",
script, data_dict)
data_output = self.start_script(script['code'], data_dict)
LOG.debug("Result [%s] for processing with pyscript [%s] with "
"data [%s].", data_output, script, data_dict)
data = dataframe.DataFrame.from_dict(data_output)
return data return data

80
cloudkitty/tests/gabbi/gabbits/v1-storage.yaml
View File

@ -75,8 +75,8 @@ tests:
$.dataframes[0].resources[0].volume: "1" $.dataframes[0].resources[0].volume: "1"
$.dataframes[0].resources[0].rating: "1.337" $.dataframes[0].resources[0].rating: "1.337"
$.dataframes[0].resources[0].service: "cpu" $.dataframes[0].resources[0].service: "cpu"
$.dataframes[0].resources[0].desc.dummy: 'True' $.dataframes[0].resources[0].desc.dummy: True
$.dataframes[0].resources[0].desc.fake_meta: '1.0' $.dataframes[0].resources[0].desc.fake_meta: 1.0
$.dataframes[1].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375" $.dataframes[1].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
$.dataframes[1].begin: "2015-01-04T13:00:00" $.dataframes[1].begin: "2015-01-04T13:00:00"
$.dataframes[1].end: "2015-01-04T14:00:00" $.dataframes[1].end: "2015-01-04T14:00:00"
@ -84,8 +84,8 @@ tests:
$.dataframes[1].resources[0].volume: "1" $.dataframes[1].resources[0].volume: "1"
$.dataframes[1].resources[0].rating: "1.337" $.dataframes[1].resources[0].rating: "1.337"
$.dataframes[1].resources[0].service: "cpu" $.dataframes[1].resources[0].service: "cpu"
$.dataframes[1].resources[0].desc.dummy: 'True' $.dataframes[1].resources[0].desc.dummy: True
$.dataframes[1].resources[0].desc.fake_meta: '1.0' $.dataframes[1].resources[0].desc.fake_meta: 1.0
$.dataframes[2].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375" $.dataframes[2].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
$.dataframes[2].begin: "2015-01-04T13:00:00" $.dataframes[2].begin: "2015-01-04T13:00:00"
$.dataframes[2].end: "2015-01-04T14:00:00" $.dataframes[2].end: "2015-01-04T14:00:00"
@ -93,8 +93,8 @@ tests:
$.dataframes[2].resources[0].volume: "1" $.dataframes[2].resources[0].volume: "1"
$.dataframes[2].resources[0].rating: "0.121" $.dataframes[2].resources[0].rating: "0.121"
$.dataframes[2].resources[0].service: "image.size" $.dataframes[2].resources[0].service: "image.size"
$.dataframes[2].resources[0].desc.dummy: 'True' $.dataframes[2].resources[0].desc.dummy: True
$.dataframes[2].resources[0].desc.fake_meta: '1.0' $.dataframes[2].resources[0].desc.fake_meta: 1.0
$.dataframes[3].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375" $.dataframes[3].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
$.dataframes[3].begin: "2015-01-04T13:00:00" $.dataframes[3].begin: "2015-01-04T13:00:00"
$.dataframes[3].end: "2015-01-04T14:00:00" $.dataframes[3].end: "2015-01-04T14:00:00"
@ -102,8 +102,8 @@ tests:
$.dataframes[3].resources[0].volume: "1" $.dataframes[3].resources[0].volume: "1"
$.dataframes[3].resources[0].rating: "0.121" $.dataframes[3].resources[0].rating: "0.121"
$.dataframes[3].resources[0].service: "image.size" $.dataframes[3].resources[0].service: "image.size"
$.dataframes[3].resources[0].desc.dummy: 'True' $.dataframes[3].resources[0].desc.dummy: True
$.dataframes[3].resources[0].desc.fake_meta: '1.0' $.dataframes[3].resources[0].desc.fake_meta: 1.0
- name: fetch data for the second tenant - name: fetch data for the second tenant
url: /v1/storage/dataframes url: /v1/storage/dataframes
@ -121,8 +121,8 @@ tests:
$.dataframes[0].resources[0].volume: "1" $.dataframes[0].resources[0].volume: "1"
$.dataframes[0].resources[0].rating: "1.337" $.dataframes[0].resources[0].rating: "1.337"
$.dataframes[0].resources[0].service: "cpu" $.dataframes[0].resources[0].service: "cpu"
$.dataframes[0].resources[0].desc.dummy: 'True' $.dataframes[0].resources[0].desc.dummy: True
$.dataframes[0].resources[0].desc.fake_meta: '1.0' $.dataframes[0].resources[0].desc.fake_meta: 1.0
$.dataframes[1].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e" $.dataframes[1].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
$.dataframes[1].begin: "2015-01-04T13:00:00" $.dataframes[1].begin: "2015-01-04T13:00:00"
$.dataframes[1].end: "2015-01-04T14:00:00" $.dataframes[1].end: "2015-01-04T14:00:00"
@ -130,8 +130,8 @@ tests:
$.dataframes[1].resources[0].volume: "1" $.dataframes[1].resources[0].volume: "1"
$.dataframes[1].resources[0].rating: "1.337" $.dataframes[1].resources[0].rating: "1.337"
$.dataframes[1].resources[0].service: "cpu" $.dataframes[1].resources[0].service: "cpu"
$.dataframes[1].resources[0].desc.dummy: 'True' $.dataframes[1].resources[0].desc.dummy: True
$.dataframes[1].resources[0].desc.fake_meta: '1.0' $.dataframes[1].resources[0].desc.fake_meta: 1.0
$.dataframes[2].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e" $.dataframes[2].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
$.dataframes[2].begin: "2015-01-04T13:00:00" $.dataframes[2].begin: "2015-01-04T13:00:00"
$.dataframes[2].end: "2015-01-04T14:00:00" $.dataframes[2].end: "2015-01-04T14:00:00"
@ -139,8 +139,8 @@ tests:
$.dataframes[2].resources[0].volume: "1" $.dataframes[2].resources[0].volume: "1"
$.dataframes[2].resources[0].rating: "0.121" $.dataframes[2].resources[0].rating: "0.121"
$.dataframes[2].resources[0].service: "image.size" $.dataframes[2].resources[0].service: "image.size"
$.dataframes[2].resources[0].desc.dummy: 'True' $.dataframes[2].resources[0].desc.dummy: True
$.dataframes[2].resources[0].desc.fake_meta: '1.0' $.dataframes[2].resources[0].desc.fake_meta: 1.0
$.dataframes[3].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e" $.dataframes[3].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
$.dataframes[3].begin: "2015-01-04T13:00:00" $.dataframes[3].begin: "2015-01-04T13:00:00"
$.dataframes[3].end: "2015-01-04T14:00:00" $.dataframes[3].end: "2015-01-04T14:00:00"
@ -148,8 +148,8 @@ tests:
$.dataframes[3].resources[0].volume: "1" $.dataframes[3].resources[0].volume: "1"
$.dataframes[3].resources[0].rating: "0.121" $.dataframes[3].resources[0].rating: "0.121"
$.dataframes[3].resources[0].service: "image.size" $.dataframes[3].resources[0].service: "image.size"
$.dataframes[3].resources[0].desc.dummy: 'True' $.dataframes[3].resources[0].desc.dummy: True
$.dataframes[3].resources[0].desc.fake_meta: '1.0' $.dataframes[3].resources[0].desc.fake_meta: 1.0
- name: fetch data for multiple tenants - name: fetch data for multiple tenants
@ -167,8 +167,8 @@ tests:
$.dataframes[0].resources[0].volume: "1" $.dataframes[0].resources[0].volume: "1"
$.dataframes[0].resources[0].rating: "1.337" $.dataframes[0].resources[0].rating: "1.337"
$.dataframes[0].resources[0].service: "cpu" $.dataframes[0].resources[0].service: "cpu"
$.dataframes[0].resources[0].desc.dummy: 'True' $.dataframes[0].resources[0].desc.dummy: True
$.dataframes[0].resources[0].desc.fake_meta: '1.0' $.dataframes[0].resources[0].desc.fake_meta: 1.0
$.dataframes[1].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e" $.dataframes[1].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
$.dataframes[1].begin: "2015-01-04T13:00:00" $.dataframes[1].begin: "2015-01-04T13:00:00"
$.dataframes[1].end: "2015-01-04T14:00:00" $.dataframes[1].end: "2015-01-04T14:00:00"
@ -176,8 +176,8 @@ tests:
$.dataframes[1].resources[0].volume: "1" $.dataframes[1].resources[0].volume: "1"
$.dataframes[1].resources[0].rating: "1.337" $.dataframes[1].resources[0].rating: "1.337"
$.dataframes[1].resources[0].service: "cpu" $.dataframes[1].resources[0].service: "cpu"
$.dataframes[1].resources[0].desc.dummy: 'True' $.dataframes[1].resources[0].desc.dummy: True
$.dataframes[1].resources[0].desc.fake_meta: '1.0' $.dataframes[1].resources[0].desc.fake_meta: 1.0
$.dataframes[2].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e" $.dataframes[2].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
$.dataframes[2].begin: "2015-01-04T13:00:00" $.dataframes[2].begin: "2015-01-04T13:00:00"
$.dataframes[2].end: "2015-01-04T14:00:00" $.dataframes[2].end: "2015-01-04T14:00:00"
@ -185,8 +185,8 @@ tests:
$.dataframes[2].resources[0].volume: "1" $.dataframes[2].resources[0].volume: "1"
$.dataframes[2].resources[0].rating: "0.121" $.dataframes[2].resources[0].rating: "0.121"
$.dataframes[2].resources[0].service: "image.size" $.dataframes[2].resources[0].service: "image.size"
$.dataframes[2].resources[0].desc.dummy: 'True' $.dataframes[2].resources[0].desc.dummy: True
$.dataframes[2].resources[0].desc.fake_meta: '1.0' $.dataframes[2].resources[0].desc.fake_meta: 1.0
$.dataframes[3].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e" $.dataframes[3].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
$.dataframes[3].begin: "2015-01-04T13:00:00" $.dataframes[3].begin: "2015-01-04T13:00:00"
$.dataframes[3].end: "2015-01-04T14:00:00" $.dataframes[3].end: "2015-01-04T14:00:00"
@ -194,8 +194,8 @@ tests:
$.dataframes[3].resources[0].volume: "1" $.dataframes[3].resources[0].volume: "1"
$.dataframes[3].resources[0].rating: "0.121" $.dataframes[3].resources[0].rating: "0.121"
$.dataframes[3].resources[0].service: "image.size" $.dataframes[3].resources[0].service: "image.size"
$.dataframes[3].resources[0].desc.dummy: 'True' $.dataframes[3].resources[0].desc.dummy: True
$.dataframes[3].resources[0].desc.fake_meta: '1.0' $.dataframes[3].resources[0].desc.fake_meta: 1.0
$.dataframes[0].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375" $.dataframes[0].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
$.dataframes[0].begin: "2015-01-04T13:00:00" $.dataframes[0].begin: "2015-01-04T13:00:00"
$.dataframes[0].end: "2015-01-04T14:00:00" $.dataframes[0].end: "2015-01-04T14:00:00"
@ -203,8 +203,8 @@ tests:
$.dataframes[0].resources[0].volume: "1" $.dataframes[0].resources[0].volume: "1"
$.dataframes[0].resources[0].rating: "1.337" $.dataframes[0].resources[0].rating: "1.337"
$.dataframes[0].resources[0].service: "cpu" $.dataframes[0].resources[0].service: "cpu"
$.dataframes[0].resources[0].desc.dummy: 'True' $.dataframes[0].resources[0].desc.dummy: True
$.dataframes[0].resources[0].desc.fake_meta: '1.0' $.dataframes[0].resources[0].desc.fake_meta: 1.0
$.dataframes[1].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375" $.dataframes[1].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
$.dataframes[1].begin: "2015-01-04T13:00:00" $.dataframes[1].begin: "2015-01-04T13:00:00"
$.dataframes[1].end: "2015-01-04T14:00:00" $.dataframes[1].end: "2015-01-04T14:00:00"
@ -212,8 +212,8 @@ tests:
$.dataframes[1].resources[0].volume: "1" $.dataframes[1].resources[0].volume: "1"
$.dataframes[1].resources[0].rating: "1.337" $.dataframes[1].resources[0].rating: "1.337"
$.dataframes[1].resources[0].service: "cpu" $.dataframes[1].resources[0].service: "cpu"
$.dataframes[1].resources[0].desc.dummy: 'True' $.dataframes[1].resources[0].desc.dummy: True
$.dataframes[1].resources[0].desc.fake_meta: '1.0' $.dataframes[1].resources[0].desc.fake_meta: 1.0
$.dataframes[2].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375" $.dataframes[2].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
$.dataframes[2].begin: "2015-01-04T13:00:00" $.dataframes[2].begin: "2015-01-04T13:00:00"
$.dataframes[2].end: "2015-01-04T14:00:00" $.dataframes[2].end: "2015-01-04T14:00:00"
@ -221,8 +221,8 @@ tests:
$.dataframes[2].resources[0].volume: "1" $.dataframes[2].resources[0].volume: "1"
$.dataframes[2].resources[0].rating: "0.121" $.dataframes[2].resources[0].rating: "0.121"
$.dataframes[2].resources[0].service: "image.size" $.dataframes[2].resources[0].service: "image.size"
$.dataframes[2].resources[0].desc.dummy: 'True' $.dataframes[2].resources[0].desc.dummy: True
$.dataframes[2].resources[0].desc.fake_meta: '1.0' $.dataframes[2].resources[0].desc.fake_meta: 1.0
$.dataframes[3].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375" $.dataframes[3].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
$.dataframes[3].begin: "2015-01-04T13:00:00" $.dataframes[3].begin: "2015-01-04T13:00:00"
$.dataframes[3].end: "2015-01-04T14:00:00" $.dataframes[3].end: "2015-01-04T14:00:00"
@ -230,8 +230,8 @@ tests:
$.dataframes[3].resources[0].volume: "1" $.dataframes[3].resources[0].volume: "1"
$.dataframes[3].resources[0].rating: "0.121" $.dataframes[3].resources[0].rating: "0.121"
$.dataframes[3].resources[0].service: "image.size" $.dataframes[3].resources[0].service: "image.size"
$.dataframes[3].resources[0].desc.dummy: 'True' $.dataframes[3].resources[0].desc.dummy: True
$.dataframes[3].resources[0].desc.fake_meta: '1.0' $.dataframes[3].resources[0].desc.fake_meta: 1.0
- name: fetch data filtering on cpu service and tenant - name: fetch data filtering on cpu service and tenant
url: /v1/storage/dataframes url: /v1/storage/dataframes
@ -250,8 +250,8 @@ tests:
$.dataframes[0].resources[0].volume: "1" $.dataframes[0].resources[0].volume: "1"
$.dataframes[0].resources[0].rating: "1.337" $.dataframes[0].resources[0].rating: "1.337"
$.dataframes[0].resources[0].service: "cpu" $.dataframes[0].resources[0].service: "cpu"
$.dataframes[0].resources[0].desc.dummy: 'True' $.dataframes[0].resources[0].desc.dummy: True
$.dataframes[0].resources[0].desc.fake_meta: '1.0' $.dataframes[0].resources[0].desc.fake_meta: 1.0
$.dataframes[1].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e" $.dataframes[1].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
$.dataframes[1].begin: "2015-01-04T13:00:00" $.dataframes[1].begin: "2015-01-04T13:00:00"
$.dataframes[1].end: "2015-01-04T14:00:00" $.dataframes[1].end: "2015-01-04T14:00:00"
@ -259,8 +259,8 @@ tests:
$.dataframes[1].resources[0].volume: "1" $.dataframes[1].resources[0].volume: "1"
$.dataframes[1].resources[0].rating: "1.337" $.dataframes[1].resources[0].rating: "1.337"
$.dataframes[1].resources[0].service: "cpu" $.dataframes[1].resources[0].service: "cpu"
$.dataframes[1].resources[0].desc.dummy: 'True' $.dataframes[1].resources[0].desc.dummy: True
$.dataframes[1].resources[0].desc.fake_meta: '1.0' $.dataframes[1].resources[0].desc.fake_meta: 1.0
- name: fetch data filtering on image service and tenant - name: fetch data filtering on image service and tenant
url: /v1/storage/dataframes url: /v1/storage/dataframes
@ -279,8 +279,8 @@ tests:
$.dataframes[0].resources[0].volume: "1" $.dataframes[0].resources[0].volume: "1"
$.dataframes[0].resources[0].rating: "0.121" $.dataframes[0].resources[0].rating: "0.121"
$.dataframes[0].resources[0].service: "image.size" $.dataframes[0].resources[0].service: "image.size"
$.dataframes[0].resources[0].desc.dummy: 'True' $.dataframes[0].resources[0].desc.dummy: True
$.dataframes[0].resources[0].desc.fake_meta: '1.0' $.dataframes[0].resources[0].desc.fake_meta: 1.0
$.dataframes[1].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e" $.dataframes[1].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
$.dataframes[1].begin: "2015-01-04T13:00:00" $.dataframes[1].begin: "2015-01-04T13:00:00"
$.dataframes[1].end: "2015-01-04T14:00:00" $.dataframes[1].end: "2015-01-04T14:00:00"
@ -288,8 +288,8 @@ tests:
$.dataframes[1].resources[0].volume: "1" $.dataframes[1].resources[0].volume: "1"
$.dataframes[1].resources[0].rating: "0.121" $.dataframes[1].resources[0].rating: "0.121"
$.dataframes[1].resources[0].service: "image.size" $.dataframes[1].resources[0].service: "image.size"
$.dataframes[1].resources[0].desc.dummy: 'True' $.dataframes[1].resources[0].desc.dummy: True
$.dataframes[1].resources[0].desc.fake_meta: '1.0' $.dataframes[1].resources[0].desc.fake_meta: 1.0
- name: fetch data filtering on service with no data and tenant - name: fetch data filtering on service with no data and tenant
url: /v1/storage/dataframes url: /v1/storage/dataframes

296
cloudkitty/tests/test_pyscripts.py
View File

@ -21,64 +21,85 @@ import zlib
from oslo_utils import uuidutils from oslo_utils import uuidutils
from cloudkitty import dataframe
from cloudkitty.rating import pyscripts from cloudkitty.rating import pyscripts
from cloudkitty.rating.pyscripts.db import api from cloudkitty.rating.pyscripts.db import api
from cloudkitty import tests from cloudkitty import tests
from dateutil import parser
FAKE_UUID = '6c1b8a30-797f-4b7e-ad66-9879b79059fb' FAKE_UUID = '6c1b8a30-797f-4b7e-ad66-9879b79059fb'
CK_RESOURCES_DATA = [{ CK_RESOURCES_DATA = {
"period": { "period": {
"begin": "2014-10-01T00:00:00", "begin": "2014-10-01T00:00:00",
"end": "2014-10-01T01:00:00"}, "end": "2014-10-01T01:00:00"},
"usage": { "usage": {
"instance_status": [
dataframe.DataPoint(
"instance", 1, 0,
{"availability_zone": "nova",
"flavor": "m1.ultra",
"image_id": "f5600101-8fa2-4864-899e-ebcb7ed6b568",
"memory": "64",
"name": "prod1",
"project_id": "f266f30b11f246b589fd266f85eeec39",
"user_id": "55b3379b949243009ee96972fbf51ed1",
"vcpus": "1"
},
{"farm": "prod"}),
dataframe.DataPoint(
"instance", 1, 0,
{"availability_zone": "nova",
"flavor": "m1.not_so_ultra",
"image_id": "f5600101-8fa2-4864-899e-ebcb7ed6b568",
"memory": "64",
"name": "prod1",
"project_id": "f266f30b11f246b589fd266f85eeec39",
"user_id": "55b3379b949243009ee96972fbf51ed1",
"vcpus": "1"
},
{"farm": "prod"})],
"compute": [ "compute": [
{ dataframe.DataPoint(
"desc": { "instance", 1, 0,
"availability_zone": "nova", {"availability_zone": "nova",
"flavor": "m1.nano", "flavor": "m1.nano",
"image_id": "f5600101-8fa2-4864-899e-ebcb7ed6b568", "image_id": "f5600101-8fa2-4864-899e-ebcb7ed6b568",
"memory": "64", "memory": "64",
"metadata": { "name": "prod1",
"farm": "prod"}, "project_id": "f266f30b11f246b589fd266f85eeec39",
"name": "prod1", "user_id": "55b3379b949243009ee96972fbf51ed1",
"project_id": "f266f30b11f246b589fd266f85eeec39", "vcpus": "1"
"user_id": "55b3379b949243009ee96972fbf51ed1", },
"vcpus": "1"}, {"farm": "prod"}),
"vol": { dataframe.DataPoint(
"qty": 1, "instance", 2, 0,
"unit": "instance"} {"availability_zone": "nova",
}, "flavor": "m1.tiny",
{ "image_id": "a41fba37-2429-4f15-aa00-b5bc4bf557bf",
"desc": { "memory": "512",
"availability_zone": "nova", "name": "dev1",
"flavor": "m1.tiny", "project_id": "f266f30b11f246b589fd266f85eeec39",
"image_id": "a41fba37-2429-4f15-aa00-b5bc4bf557bf", "user_id": "55b3379b949243009ee96972fbf51ed1",
"memory": "512", "vcpus": "1"
"metadata": { },
"farm": "dev"}, {"farm": "dev"}),
"name": "dev1", dataframe.DataPoint(
"project_id": "f266f30b11f246b589fd266f85eeec39", "instance", 1, 0,
"user_id": "55b3379b949243009ee96972fbf51ed1", {"availability_zone": "nova",
"vcpus": "1"}, "flavor": "m1.nano",
"vol": { "image_id": "a41fba37-2429-4f15-aa00-b5bc4bf557bf",
"qty": 2, "memory": "64",
"unit": "instance"}}, "name": "dev2",
{ "project_id": "f266f30b11f246b589fd266f85eeec39",
"desc": { "user_id": "55b3379b949243009ee96972fbf51ed1",
"availability_zone": "nova", "vcpus": "1"
"flavor": "m1.nano", },
"image_id": "a41fba37-2429-4f15-aa00-b5bc4bf557bf", {"farm": "dev"}),
"memory": "64", ]
"metadata": { }
"farm": "dev"}, }
"name": "dev2",
"project_id": "f266f30b11f246b589fd266f85eeec39",
"user_id": "55b3379b949243009ee96972fbf51ed1",
"vcpus": "1"},
"vol": {
"qty": 1,
"unit": "instance"}}]}}]
TEST_CODE1 = 'a = 1'.encode('utf-8') TEST_CODE1 = 'a = 1'.encode('utf-8')
TEST_CODE1_CHECKSUM = hashlib.sha512(TEST_CODE1).hexdigest() TEST_CODE1_CHECKSUM = hashlib.sha512(TEST_CODE1).hexdigest()
@ -90,14 +111,89 @@ TEST_CODE3_CHECKSUM = hashlib.sha512(TEST_CODE3).hexdigest()
COMPLEX_POLICY1 = """ COMPLEX_POLICY1 = """
import decimal import decimal
usage_data = data['usage']
for service in usage_data.keys():
if service == 'compute':
all_points = usage_data.get(service, [])
for resource in all_points:
if resource['groupby'].get('flavor') == 'm1.nano':
resource['rating'] = {
'price': decimal.Decimal(2.0)}
if service == 'instance_status':
all_points = usage_data.get(service, [])
for resource in all_points:
if resource['groupby'].get('flavor') == 'm1.ultra':
resource['rating'] = {
'price': decimal.Decimal(
resource['groupby'].get(
'memory')) * decimal.Decimal(1.5)}
""".encode('utf-8')
for period in data:
for service, resources in period['usage'].items(): DOCUMENTATION_RATING_POLICY = """
if service == 'compute': import decimal
for resource in resources:
if resource['desc'].get('flavor') == 'm1.nano':
resource['rating'] = { # Price for each flavor. These are equivalent to hashmap field mappings.
'price': decimal.Decimal(1.0)} flavors = {
'm1.micro': decimal.Decimal(0.65),
'm1.nano': decimal.Decimal(0.35),
'm1.large': decimal.Decimal(2.67)
}
# Price per MB / GB for images and volumes. These are equivalent to
# hashmap service mappings.
image_mb_price = decimal.Decimal(0.002)
volume_gb_price = decimal.Decimal(0.35)
# These functions return the price of a service usage on a collect period.
# The price is always equivalent to the price per unit multiplied by
# the quantity.
def get_compute_price(item):
flavor_name = item['groupby']['flavor']
if not flavor_name in flavors:
return 0
else:
return (decimal.Decimal(item['vol']['qty']) * flavors[flavor_name])
def get_image_price(item):
if not item['vol']['qty']:
return 0
else:
return decimal.Decimal(item['vol']['qty']) * image_mb_price
def get_volume_price(item):
if not item['vol']['qty']:
return 0
else:
return decimal.Decimal(item['vol']['qty']) * volume_gb_price
# Mapping each service to its price calculation function
services = {
'compute': get_compute_price,
'volume': get_volume_price,
'image': get_image_price
}
def process(data):
# The 'data' is a dictionary with the usage entries for each service for
# each given period.
usage_data = data['usage']
for service_name, service_data in usage_data.items():
# Do not calculate the price if the service has no
# price calculation function
if service_name in services.keys():
# A service can have several items. For example,
# each running instance is an item of the compute service
for item in service_data:
item['rating'] = {'price': services[service_name](item)}
return data
# 'data' is passed as a global variable. The script is supposed to set the
# 'rating' element of each item in each service
data = process(data)
""".encode('utf-8') """.encode('utf-8')
@ -109,6 +205,11 @@ class PyScriptsRatingTest(tests.TestCase):
self._db_api.get_migration().upgrade('head') self._db_api.get_migration().upgrade('head')
self._pyscripts = pyscripts.PyScripts(self._tenant_id) self._pyscripts = pyscripts.PyScripts(self._tenant_id)
self.dataframe_for_tests = dataframe.DataFrame(
parser.parse(CK_RESOURCES_DATA['period']['begin']),
parser.parse(CK_RESOURCES_DATA['period']['end']),
CK_RESOURCES_DATA['usage'])
# Scripts tests # Scripts tests
@mock.patch.object(uuidutils, 'generate_uuid', @mock.patch.object(uuidutils, 'generate_uuid',
return_value=FAKE_UUID) return_value=FAKE_UUID)
@ -295,16 +396,87 @@ class PyScriptsRatingTest(tests.TestCase):
self._db_api.create_script('policy1', TEST_CODE1) self._db_api.create_script('policy1', TEST_CODE1)
self._db_api.create_script('policy2', TEST_CODE3) self._db_api.create_script('policy2', TEST_CODE3)
self._pyscripts.reload_config() self._pyscripts.reload_config()
self.assertRaises(NameError, self._pyscripts.process, {})
self.assertEqual(2, len(self._pyscripts._scripts))
self.assertRaises(NameError, self._pyscripts.process,
self.dataframe_for_tests)
# Processing # Processing
def test_process_rating(self): def test_process_rating(self):
self._db_api.create_script('policy1', COMPLEX_POLICY1) self._db_api.create_script('policy1', COMPLEX_POLICY1)
self._pyscripts.reload_config() self._pyscripts.reload_config()
actual_data = copy.deepcopy(CK_RESOURCES_DATA)
expected_data = copy.deepcopy(CK_RESOURCES_DATA) data_output = self._pyscripts.process(self.dataframe_for_tests)
compute_list = expected_data[0]['usage']['compute'] self.assertIsInstance(data_output, dataframe.DataFrame)
compute_list[0]['rating'] = {'price': decimal.Decimal('1')}
compute_list[2]['rating'] = {'price': decimal.Decimal('1')} dict_output = data_output.as_dict()
self._pyscripts.process(actual_data) for point in dict_output['usage']['compute']:
self.assertEqual(expected_data, actual_data) if point['groupby'].get('flavor') == 'm1.nano':
self.assertEqual(
decimal.Decimal('2'), point['rating']['price'])
else:
self.assertEqual(
decimal.Decimal('0'), point['rating']['price'])
for point in dict_output['usage']['instance_status']:
if point['groupby'].get('flavor') == 'm1.ultra':
self.assertEqual(
decimal.Decimal('96'), point['rating']['price'])
else:
self.assertEqual(
decimal.Decimal('0'), point['rating']['price'])
# Processing
def test_process_rating_with_documentation_rules(self):
self._db_api.create_script('policy1', DOCUMENTATION_RATING_POLICY)
self._pyscripts.reload_config()
dataframe_for_tests = copy.deepcopy(self.dataframe_for_tests)
dataframe_for_tests.add_point(
dataframe.DataPoint("GB", 5, 0, {"tag": "A"}, {}), "image")
dataframe_for_tests.add_point(
dataframe.DataPoint("GB", 15, 0, {"tag": "B"}, {}), "image")
dataframe_for_tests.add_point(
dataframe.DataPoint("GB", 500, 0, {"tag": "D"}, {}), "volume")
dataframe_for_tests.add_point(
dataframe.DataPoint("GB", 80, 0, {"tag": "E"}, {}), "volume")
data_output = self._pyscripts.process(dataframe_for_tests)
self.assertIsInstance(data_output, dataframe.DataFrame)
dict_output = data_output.as_dict()
for point in dict_output['usage']['compute']:
if point['groupby'].get('flavor') == 'm1.nano':
self.assertEqual(
decimal.Decimal('0.3499999999999999777955395075'),
point['rating']['price'])
else:
self.assertEqual(
decimal.Decimal('0'), point['rating']['price'])
for point in dict_output['usage']['instance_status']:
if point['groupby'].get('flavor') == 'm1.ultra':
self.assertEqual(
decimal.Decimal('0'), point['rating']['price'])
else:
self.assertEqual(
decimal.Decimal('0'), point['rating']['price'])
for point in dict_output['usage']['image']:
if point['groupby'].get('tag') == 'A':
self.assertEqual(
decimal.Decimal('0.01000000000000000020816681712'),
point['rating']['price'])
else:
self.assertEqual(
decimal.Decimal('0.03000000000000000062450045135'),
point['rating']['price'])
for point in dict_output['usage']['volume']:
if point['groupby'].get('tag') == 'D':
self.assertEqual(
decimal.Decimal('174.9999999999999888977697537'),
point['rating']['price'])
else:
self.assertEqual(
decimal.Decimal('27.99999999999999822364316060'),
point['rating']['price'])

24
doc/source/user/rating/pyscripts.rst
View File

@ -74,18 +74,18 @@ Processing the data
.. code-block:: python .. code-block:: python
def process(data): def process(data):
# The 'data' parameter is a list of dictionaries containing a # The 'data' is a dictionary with the usage entries for each service
# "usage" and a "period" field # in a given period.
for d in data: usage_data = data['usage']
usage = d['usage']
for service_name, service_data in usage.items(): for service_name, service_data in usage_data.items():
# Do not calculate the price if the service has no # Do not calculate the price if the service has no
# price calculation function # price calculation function
if service_name in services.keys(): if service_name in services.keys():
# A service can have several items. For example, # A service can have several items. For example,
# each running instance is an item of the compute service # each running instance is an item of the compute service
for item in service_data: for item in service_data:
item['rating'] = {'price': services[service_name](item)} item['rating'] = {'price': services[service_name](item)}
return data return data

4
releasenotes/notes/fix_py_scripts-fd9ab52c92263844.yaml Normal file
View File

@ -0,0 +1,4 @@
---
fixes:
- |
Fix failure to process rating using the PyScripts rating module.