openstack
/
congress
Code Issues Proposed changes

Automatically load library policy files at start 

harness.py loads library policy from disk files to DB

uniqueness constraint added on library policy name.

devstack plugin updated to install library policy files
to default location

updated congress stand-alone install instruction

Partially implements: blueprint policy-library
Closes-Bug: 1693619
Closes-Bug: 1693672

Change-Id: I51097081f6576755751231feb5ed2b0be642d91e
This commit is contained in:
Eric Kao 2017-06-19 17:03:13 -07:00
parent 48765b818f
commit cd9aa33451
19 changed files with 220 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
README.rst
View File

@ -185,7 +185,16 @@ Configure Congress (Assume you put config files in /etc/congress)
$ sudo cp etc/api-paste.ini /etc/congress
$ sudo cp etc/policy.json /etc/congress
Set-up Policy Library [optional]
This step copies the bundled collection Congress policies into the Congress
policy library for easy activation by an administrator. The policies in the
library do not become active until explicitly activated by an administrator.
The step may be skipped if you do not want to load the bundled policies into
the policy library.
.. code-block:: console
$ sudo cp -r library /etc/congress/.
Generate a configuration file as outlined in the Configuration Options section
of the :ref:`Deployment <deployment>` document. Note: you may have to run the command with sudo.

73
congress/api/library_policy_model.py
View File

@ -17,9 +17,6 @@ from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
import json
import jsonschema
from oslo_log import log as logging
from congress.api import base
@ -97,7 +94,6 @@ class LibraryPolicyModel(base.APIModel):
(num, desc) = error_codes.get('policy_id_must_not_be_provided')
raise webservice.DataModelException(num, desc)
self._validate_policy_item(item)
try:
# Note(thread-safety): blocking call
policy_metadata = self.invoke_rpc(
@ -144,8 +140,6 @@ class LibraryPolicyModel(base.APIModel):
Raises:
KeyError: Item with specified id_ not present.
"""
self._validate_policy_item(item)
# Note(thread-safety): blocking call
try:
return self.invoke_rpc(base.LIBRARY_SERVICE_ID,
@ -154,70 +148,3 @@ class LibraryPolicyModel(base.APIModel):
'policy_dict': item})
except exception.CongressException as e:
raise webservice.DataModelException.create(e)
def _validate_policy_item(self, item):
schema_json = '''
{
"id": "PolicyProperties",
"title": "Policy Properties",
"type": "object",
"required": ["name", "rules"],
"properties": {
"name": {
"title": "Policy unique name",
"type": "string",
"minLength": 1,
"maxLength": 255
},
"description": {
"title": "Policy description",
"type": "string"
},
"kind": {
"title": "Policy kind",
"type": "string",
"enum": ["database", "nonrecursive", "action", "materialized",
"delta", "datasource"]
},
"abbreviation": {
"title": "Policy name abbreviation",
"type": "string",
"minLength": 1,
"maxLength": 5
},
"rules": {
"title": "collection of rules",
"type": "array",
"items": {
"type": "object",
"properties": {
"PolicyRule": {
"title": "Policy rule",
"type": "object",
"required": ["rule"],
"properties": {
"rule": {
"title": "Rule definition following policy grammar",
"type": "string"
},
"name": {
"title": "User-friendly name",
"type": "string"
},
"comment": {
"title": "User-friendly comment",
"type": "string"
}
}
}
}
}
}
}
}
'''
try:
jsonschema.validate(item, json.loads(schema_json))
except jsonschema.exceptions.ValidationError as ve:
raise webservice.DataModelException(
1000, 'Input item violates JSON Schema', data=str(ve))

2
congress/common/config.py
View File

@ -75,6 +75,8 @@ core_opts = [
cfg.BoolOpt('replicated_policy_engine', default=False,
help='Set the flag to use congress with replicated policy '
'engines.'),
cfg.StrOpt('policy_library_path', default='/etc/congress/library',
help=_('The directory containing library policy files.')),
cfg.BoolOpt('distributed_architecture',
deprecated_for_removal=True,
deprecated_reason='distributed architecture is now the only '

27
congress/db/db_library_policies.py
View File

@ -18,6 +18,7 @@ from __future__ import absolute_import
import json
from oslo_db import exception as oslo_db_exc
import sqlalchemy as sa
from sqlalchemy.orm import exc as db_exc
@ -26,9 +27,9 @@ from congress.db import model_base
class LibraryPolicy(model_base.BASE, model_base.HasId):
__tablename__ = 'librarypolicies'
__tablename__ = 'library_policies'
name = sa.Column(sa.String(255), nullable=False)
name = sa.Column(sa.String(255), nullable=False, unique=True)
abbreviation = sa.Column(sa.String(5), nullable=False)
description = sa.Column(sa.Text(), nullable=False)
kind = sa.Column(sa.Text(), nullable=False)
@ -60,15 +61,19 @@ class LibraryPolicy(model_base.BASE, model_base.HasId):
def add_policy(policy_dict, session=None):
session = session or db.get_session()
with session.begin(subtransactions=True):
new_row = LibraryPolicy(
name=policy_dict['name'],
abbreviation=policy_dict['abbreviation'],
description=policy_dict['description'],
kind=policy_dict['kind'],
rules=json.dumps(policy_dict['rules']))
session.add(new_row)
return new_row
try:
with session.begin(subtransactions=True):
new_row = LibraryPolicy(
name=policy_dict['name'],
abbreviation=policy_dict['abbreviation'],
description=policy_dict['description'],
kind=policy_dict['kind'],
rules=json.dumps(policy_dict['rules']))
session.add(new_row)
return new_row
except oslo_db_exc.DBDuplicateEntry:
raise KeyError(
"Policy with name %s already exists" % policy_dict['name'])
def replace_policy(id_, policy_dict, session=None):

6
congress/db/migration/alembic_migrations/versions/c0125080d572_policy_library.py
View File

@ -31,9 +31,9 @@ import sqlalchemy as sa
def upgrade():
op.create_table(
'librarypolicies',
'library_policies',
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False, unique=True),
sa.Column('abbreviation', sa.String(length=5), nullable=False),
sa.Column('description', sa.Text(), nullable=False),
sa.Column('kind', sa.Text(), nullable=False),
@ -44,4 +44,4 @@ def upgrade():
def downgrade():
op.drop_table('librarypolicies')
op.drop_table('library_policies')

4
congress/exception.py
View File

@ -160,6 +160,10 @@ class LazyTable(BadRequest):
msg_fmt = _("table %(lazy_table)s is a lazy table and is not subscribed.")
class InvalidPolicyInput(BadRequest):
msg_fmt = _('Input policy item violates schema.')
# NOTE(thinrichs): The following represent different kinds of
# exceptions: the policy compiler and the policy runtime, respectively.
class PolicyException(CongressException):

3
congress/harness.py
View File

@ -151,6 +151,9 @@ def initialize_policy_engine(engine):
def create_policy_library_service():
"""Create policy library service."""
library = library_service.LibraryService(api_base.LIBRARY_SERVICE_ID)
# load library policies from file if none present in DB
if len(library.get_policies(include_rules=False)) == 0:
library.load_policies_from_files()
return library

118
congress/library_service/library_service.py
View File

@ -18,7 +18,12 @@ from __future__ import division
from __future__ import absolute_import
import copy
import json
import jsonschema
import os
import yaml
from oslo_config import cfg
from oslo_db import exception as db_exc
from oslo_log import log as logging
@ -38,6 +43,7 @@ class LibraryService (data_service.DataService):
def create_policy(self, policy_dict):
policy_dict = copy.deepcopy(policy_dict)
self._validate_policy_item(policy_dict)
policy_name = policy_dict['name']
# check name is valid
@ -81,6 +87,7 @@ class LibraryService (data_service.DataService):
return db_object.to_dict(include_rules=True)
def replace_policy(self, id_, policy_dict):
self._validate_policy_item(policy_dict)
policy_name = policy_dict['name']
# check name is valid
@ -101,6 +108,111 @@ class LibraryService (data_service.DataService):
id_, policy_dict=policy_dict)
return policy.to_dict()
def _validate_policy_item(self, item):
schema_json = '''
{
"id": "PolicyProperties",
"title": "Policy Properties",
"type": "object",
"required": ["name", "rules"],
"properties": {
"name": {
"title": "Policy unique name",
"type": "string",
"minLength": 1,
"maxLength": 255
},
"description": {
"title": "Policy description",
"type": "string"
},
"kind": {
"title": "Policy kind",
"type": "string",
"enum": ["database", "nonrecursive", "action", "materialized",
"delta", "datasource"]
},
"abbreviation": {
"title": "Policy name abbreviation",
"type": "string",
"minLength": 1,
"maxLength": 5
},
"rules": {
"title": "collection of rules",
"type": "array",
"items": {
"type": "object",
"properties": {
"PolicyRule": {
"title": "Policy rule",
"type": "object",
"required": ["rule"],
"properties": {
"rule": {
"title": "Rule definition following policy grammar",
"type": "string"
},
"name": {
"title": "User-friendly name",
"type": "string"
},
"comment": {
"title": "User-friendly comment",
"type": "string"
}
}
}
}
}
}
}
}
'''
try:
jsonschema.validate(item, json.loads(schema_json))
except jsonschema.exceptions.ValidationError as ve:
raise exception.InvalidPolicyInput(data=str(ve))
def load_policies_from_files(self):
def _load_library_policy_file(full_path):
with open(full_path, "r") as stream:
policies = yaml.load_all(stream)
count = 0
doc_num_in_file = 0
for policy in policies:
try:
doc_num_in_file += 1
self.create_policy(policy)
count += 1
except db_exc.DBDuplicateEntry:
LOG.debug(
'Library policy %s (number %s in file %s) already '
'exists (likely loaded by another Congress '
'instance). Skipping.',
policy.get('name', '[no name]'),
doc_num_in_file, full_path)
except exception.CongressException:
LOG.exception(
'Library policy %s could not be loaded. Skipped. '
'YAML reproduced here %s',
policy.get('name', '[no name]'),
yaml.dumps(policy))
return count
file_count = 0
policy_count = 0
for (dirpath, dirnames, filenames) in os.walk(
cfg.CONF.policy_library_path):
for filename in filenames:
count = _load_library_policy_file(
os.path.join(dirpath, filename))
if count > 0:
file_count += 1
policy_count += count
LOG.debug(
'%s library policies from %s files successfully loaded',
policy_count, file_count)
class DseLibraryServiceEndpoints(object):
"""RPC endpoints exposed by LibraryService."""
@ -108,10 +220,8 @@ class DseLibraryServiceEndpoints(object):
def __init__(self, data_service):
self.data_service = data_service
def create_policy(
self, context, policy_dict):
return self.data_service.create_policy(
policy_dict)
def create_policy(self, context, policy_dict):
return self.data_service.create_policy(policy_dict)
def get_policies(self, context, include_rules=True):
return self.data_service.get_policies(include_rules)

12
congress/tests/api/test_library_policy_model.py
View File

@ -20,6 +20,7 @@ from __future__ import absolute_import
import copy
from congress.api import webservice
from congress.db import db_library_policies
from congress.tests.api import base as api_base
from congress.tests import base
@ -32,6 +33,10 @@ class TestLibraryPolicyModel(base.SqlTestCase):
self.library_policy_model = services['api']['api-library-policy']
self.node = services['node']
self.engine = services['engine']
# clear the library policies loaded on startup
db_library_policies.delete_policies()
self._add_test_policy()
def _add_test_policy(self):
@ -96,7 +101,6 @@ class TestLibraryPolicyModel(base.SqlTestCase):
del expected_ret['rules']
policy_id, policy_obj = self.library_policy_model.add_item(test, {})
# self.assertEqual(test['id'], policy_id)
test['id'] = policy_id
self.assertEqual(test, policy_obj)
@ -108,9 +112,11 @@ class TestLibraryPolicyModel(base.SqlTestCase):
"abbreviation": "abbr",
"rules": []
}
self.library_policy_model.add_item(test, {})
# duplicate name allowed
self.assertRaises(KeyError,
self.library_policy_model.add_item, test, {})
ret = self.library_policy_model.get_items({})
self.assertEqual(len(ret['results']), 3)
self.assertEqual(len(ret['results']), 2)
def test_add_item_with_id(self):
test = {

1
congress/tests/db/test_db_library_policies.py
View File

@ -25,6 +25,7 @@ class TestDbLibraryPolicies(base.SqlTestCase):
def setUp(self):
super(TestDbLibraryPolicies, self).setUp()
db_library_policies.delete_policies() # delete preloaded policies
def test_add_policy_no_name(self):
self.assertRaises(

BIN
congress/tests/haht/test.db.clean
View File

Binary file not shown.

38
congress/tests/library_service/test_library_service.py
View File

@ -29,6 +29,7 @@ class TestLibraryService(base.SqlTestCase):
def setUp(self):
super(TestLibraryService, self).setUp()
self.library = library_service.LibraryService('lib-test')
self.library.delete_all_policies() # clear pre-loaded library policies
self.policy1 = {'name': 'policy1', 'abbreviation': 'abbr',
'kind': 'database', 'description': 'descrip',
@ -48,12 +49,12 @@ class TestLibraryService(base.SqlTestCase):
del self.policy2_meta['rules']
def test_create_policy_no_name(self):
self.assertRaises(
KeyError, self.library.create_policy, {'rules': []})
self.assertRaises(exception.InvalidPolicyInput,
self.library.create_policy, {'rules': []})
def test_create_policy_no_rules(self):
self.assertRaises(KeyError, self.library.create_policy,
{'name': 'policy1'})
self.assertRaises(exception.InvalidPolicyInput,
self.library.create_policy, {'name': 'policy1'})
def test_create_policy_bad_name(self):
self.assertRaises(exception.PolicyException,
@ -73,9 +74,10 @@ class TestLibraryService(base.SqlTestCase):
def test_create_policy_duplicate(self):
self.library.create_policy({'name': 'policy1', 'rules': []})
self.library.create_policy({'name': 'policy1', 'rules': []})
self.assertRaises(KeyError, self.library.create_policy,
{'name': 'policy1', 'rules': []})
res = self.library.get_policies()
self.assertEqual(len(res), 2)
self.assertEqual(len(res), 1)
def test_get_policy_empty(self):
res = self.library.get_policies()
@ -140,15 +142,15 @@ class TestLibraryService(base.SqlTestCase):
self.library.create_policy(
{'name': 'policy1', 'abbreviation': 'abbr', 'kind': 'database',
'description': 'descrip', 'rules': [[{'rule': 'p(x) :- q(x)',
'comment': 'test comment',
'name': 'testname'}]]})
'description': 'descrip', 'rules': [{'rule': 'p(x) :- q(x)',
'comment': 'test comment',
'name': 'testname'}]})
self.library.create_policy(
{'name': 'policy2', 'abbreviation': 'abbr', 'kind': 'database',
'description': 'descrip', 'rules': [[{'rule': 'p(x) :- q(x)',
'comment': 'test comment',
'name': 'testname'}]]})
'description': 'descrip', 'rules': [{'rule': 'p(x) :- q(x)',
'comment': 'test comment',
'name': 'testname'}]})
self.library.delete_all_policies()
res = self.library.get_policies()
@ -157,15 +159,15 @@ class TestLibraryService(base.SqlTestCase):
def test_replace_policy(self):
policy1 = self.library.create_policy(
{'name': 'policy1', 'abbreviation': 'abbr', 'kind': 'database',
'description': 'descrip', 'rules': [[{'rule': 'p(x) :- q(x)',
'comment': 'test comment',
'name': 'testname'}]]})
'description': 'descrip', 'rules': [{'rule': 'p(x) :- q(x)',
'comment': 'test comment',
'name': 'testname'}]})
policy2 = self.library.create_policy(
{'name': 'policy2', 'abbreviation': 'abbr', 'kind': 'database',
'description': 'descrip', 'rules': [[{'rule': 'p(x) :- q(x)',
'comment': 'test comment',
'name': 'testname'}]]})
'description': 'descrip', 'rules': [{'rule': 'p(x) :- q(x)',
'comment': 'test comment',
'name': 'testname'}]})
replacement_policy = {
"name": "new_name",

4
congress/tests/test_congress.py
View File

@ -31,6 +31,7 @@ from congress.api import base as api_base
from congress.common import config
from congress.datasources import neutronv2_driver
from congress.datasources import nova_driver
from congress.db import db_library_policies
from congress.tests.api import base as tests_api_base
from congress.tests import base
from congress.tests.datasources import test_neutron_driver as test_neutron
@ -85,6 +86,9 @@ class TestCongress(BaseTestPolicyCongress):
"""Setup tests that use multiple mock neutron instances."""
super(TestCongress, self).setUp()
# clear the library policies loaded on startup
db_library_policies.delete_policies()
def tearDown(self):
super(TestCongress, self).tearDown()

4
congress_tempest_tests/tests/scenario/test_congress_basic_ops.py
View File

@ -196,6 +196,10 @@ class TestPolicyLibraryBasicOps(manager_congress.ScenarioPolicyBase):
response = self.admin_manager.congress_client.list_library_policy()
initial_state = response['results']
self.assertGreater(
len(initial_state), 0, 'library policy shows no policies, '
'indicating failed load-on-startup.')
test_policy = {
"name": "test_policy",
"description": "test policy description",

2
devstack/plugin.sh
View File

@ -51,6 +51,8 @@ function configure_congress {
cp $CONGRESS_DIR/etc/api-paste.ini $CONGRESS_API_PASTE_FILE
cp $CONGRESS_DIR/etc/policy.json $CONGRESS_POLICY_FILE
mkdir $CONGRESS_LIBRARY_DIR
cp $CONGRESS_DIR/library/* $CONGRESS_LIBRARY_DIR
# Update either configuration file
iniset $CONGRESS_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL

2
devstack/settings
View File

@ -36,6 +36,8 @@ CONGRESS_REPLICATED=${CONGRESS_REPLICATED:-False}
CONGRESS_TRANSPORT_URL=${CONGRESS_TRANSPORT_URL:-kombu+memory:////}
# Mutli process deployment
CONGRESS_MULTIPROCESS_DEPLOYMENT=${CONGRESS_MULTIPROCESS_DEPLOYMENT:-False}
# Directory path to library policy files
CONGRESS_LIBRARY_DIR=$CONGRESS_CONF_DIR/library
# File path to predefined policy and rules
CONGRESS_PREDEFINED_POLICY_FILE=${CONGRESS_PREDEFINED_POLICY_FILE:-""}

31
library/pause_disallowed_flavors.yaml
View File

@ -1,16 +1,17 @@
id: PauseBadFlavors
description: Pause any server using a flavor that is not permitted
---
name: PauseBadFlavors
description: "Pause any server using a flavor that is not permitted"
rules:
- comment: "User should customize this. Permitted flavors."
rule: permitted_flavor('m1.tiny')
- comment: "User should customize this. Permitted flavors."
rule: permitted_flavor('m1.large')
- rule: >
server_with_bad_flavor(id) :- nova:servers(id=id,flavor_id=flavor_id), nova:flavors(id=flavor_id, name=flavor),
not permitted_flavor(flavor)
- comment: "Remediation: Pause any VM that shows up in the server_with_bad_flavor table"
rule: "execute[nova:servers.pause(id)] :- server_with_bad_flavor(id), nova:servers(id,status='ACTIVE')"
-
comment: "User should customize this. Permitted flavors."
rule: permitted_flavor('m1.tiny')
-
comment: "User should customize this. Permitted flavors."
rule: permitted_flavor('m1.large')
-
rule: >
"server_with_bad_flavor(id) :- nova:servers(id=id,flavor_id=flavor_id),
nova:flavors(id=flavor_id, name=flavor), not permitted_flavor(flavor)"
-
comment: "Remediation: Pause any VM that shows up in the server_with_bad_flavor table"
rule: "execute[nova:servers.pause(id)] :- server_with_bad_flavor(id), nova:servers(id,status='ACTIVE')"

10
releasenotes/notes/load-lib-policies-a5cca19f58f9030c.yaml Normal file
View File

@ -0,0 +1,10 @@
---
prelude: >
upgrade:
- A new config option `policy_library_path` is added to the [DEFAULT]
section. The string option specifies the directory from which
Congress will load pre-written policies for easy activation later
by an administrator.
This option can be ignored if you do not want
Congress to load pre-written policies from files. Due to MySQL limitations,
the full path to each policy file cannot exceed 760 characters.

2
releasenotes/notes/policy-lib-db-656f809410706e6a.yaml
View File

@ -1,5 +1,5 @@
---
prelude: >
upgrade:
- A new database table `librarypolicies` is added;
- A new database table `library_policies` is added;
alembic migration scripts included.