x
/
tuskar
Code Issues Proposed changes

Support non-role resources in resource registry 

This change allows for resources utilized in the master seed that are
specified in the resource registry, but are not a provider resource to
be stored and saved to disk. There is no additional information required
to pass as all the data is stored while loading roles.

Previously tuskar always assumed that there was a 1:1 correspondence
between the final resource registry and the number of roles, but this is
no longer the case. The API for /v2/plans/templates has changed to
include non-role resources specified in the resource registry (CLI
operation tuskar plan-templates).

(Also, ignores tags file in git)

Change-Id: Ia15c0fdad0c036564b81f664e49fca280bc7ce7e
This commit is contained in:
Jeff Peeler 2015-03-03 10:25:48 -05:00
parent 6c95d0387c
commit d4785d9c2c
8 changed files with 187 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -32,3 +32,4 @@ ChangeLog
etc/tuskar/tuskar.conf
*.sqlite
tuskar/api/templates/tripleo-heat-templates/
tags

24
tuskar/manager/plan.py
View File

@ -22,9 +22,11 @@ from tuskar.storage.stores import DeploymentPlanStore
from tuskar.storage.stores import EnvironmentFileStore
from tuskar.storage.stores import MasterSeedStore
from tuskar.storage.stores import MasterTemplateStore
from tuskar.storage.stores import ResourceRegistryMappingStore
from tuskar.storage.stores import ResourceRegistryStore
from tuskar.storage.stores import TemplateStore
from tuskar.templates import composer
from tuskar.templates.heat import RegistryEntry
from tuskar.templates import namespace as ns_utils
from tuskar.templates import parser
from tuskar.templates import plan
@ -42,6 +44,7 @@ class PlansManager(object):
self.plan_store = DeploymentPlanStore()
self.seed_store = MasterSeedStore()
self.registry_store = ResourceRegistryStore()
self.registry_mapping_store = ResourceRegistryMappingStore()
self.template_store = TemplateStore()
self.master_template_store = MasterTemplateStore()
self.environment_store = EnvironmentFileStore()
@ -189,6 +192,18 @@ class PlansManager(object):
seed_role,
role_namespace)
# Update environment file to add top level mappings, which is made
# up of all non-role files present in the resource registry
reg_mapping = self.registry_mapping_store.list()
environment = deployment_plan.environment
for entry in parsed_registry_env.registry_entries:
# check if registry_mapping is in database, if so add to
# environment (later will become environment.yaml)
if any(x.name == entry.filename for x in reg_mapping):
additem = RegistryEntry(entry.alias, entry.filename)
environment.add_registry_entry(additem, unique=True)
# Save the updated plan.
updated = self._save_updated_plan(plan_uuid, deployment_plan)
@ -359,6 +374,11 @@ class PlansManager(object):
role.version)
files_dict[filename] = contents
# in addition to provider roles above, return non-role template files
reg_mapping = self.registry_mapping_store.list()
for entry in reg_mapping:
files_dict[entry.name] = entry.contents
return files_dict
def _find_roles(self, environment):
@ -386,7 +406,9 @@ class PlansManager(object):
role.description, role)
return tuskar_role
roles = [load_role(e) for e in environment.registry_entries]
reg_mapping = self.registry_mapping_store.list()
roles = [load_role(e) for e in environment.registry_entries
if not any(x.name == e.filename for x in reg_mapping)]
return roles
@staticmethod

22
tuskar/storage/load_roles.py
View File

@ -21,8 +21,10 @@ from os import path
from tuskar.storage.exceptions import UnknownName
from tuskar.storage.stores import MasterSeedStore
from tuskar.storage.stores import ResourceRegistryMappingStore
from tuskar.storage.stores import ResourceRegistryStore
from tuskar.storage.stores import TemplateStore
from tuskar.templates import parser
MASTER_SEED_NAME = '_master_seed'
@ -124,4 +126,24 @@ def load_roles(roles, seed_file=None, resource_registry_path=None,
else:
updated.append(RESOURCE_REGISTRY_NAME)
parsed_env = parser.parse_environment(contents)
mapping_store = ResourceRegistryMappingStore()
dirname = path.dirname(resource_registry_path)
role_paths = [r[1] for r in roles]
for entry in parsed_env.registry_entries:
complete_path = path.join(dirname, entry.filename)
# skip adding if entry is not a filename (is another alias) or
# if template has already been stored as a role
if (not entry.is_filename() or complete_path in role_paths):
continue
mapping_created, _ = _create_or_update(entry.filename,
_load_file(complete_path),
store=mapping_store)
if mapping_created:
created.append(entry.filename)
else:
updated.append(entry.filename)
return all_roles, created, updated

4
tuskar/storage/stores.py
View File

@ -225,6 +225,10 @@ class ResourceRegistryStore(_NamedStore):
object_type = "registry"
class ResourceRegistryMappingStore(_NamedStore):
object_type = "registry_mapping"
class EnvironmentFileStore(_BaseStore):
"""Environment File for Heat environment files.

20
tuskar/templates/heat.py
View File

@ -17,6 +17,8 @@ These objects were created against the HOT specification found at:
http://docs.openstack.org/developer/heat/template_guide/hot_spec.html
"""
from os import path
from tuskar.templates import namespace as ns_utils
@ -460,12 +462,19 @@ class Environment(object):
return True
return False
def add_registry_entry(self, entry):
def add_registry_entry(self, entry, unique=False):
"""Adds a registry entry to the environment.
:type entry: tuskar.templates.heat.RegistryEntry
:param unique: toggles if registry is treated as a set
:type unique: boolean
"""
self._registry_entries.append(entry)
if unique:
setentries = set(self._registry_entries)
setentries.add(entry)
self._registry_entries = list(setentries)
else:
self._registry_entries.append(entry)
def remove_registry_entry(self, entry):
"""Removes a registry entry from the environment.
@ -506,6 +515,7 @@ class RegistryEntry(object):
super(RegistryEntry, self).__init__()
self.alias = alias
self.filename = filename
# TODO(jpeeler) rename self.filename to mapping
def __str__(self):
msg = 'RegistryEntry: alias=%(alias)s, filename=%(f)s'
@ -515,6 +525,12 @@ class RegistryEntry(object):
}
return msg % data
def is_filename(self):
if ('::' in self.filename or
path.splitext(self.filename)[1] not in ('.yaml', '.yml')):
return False
return True
def _safe_strip(value):
"""Strips the value if it is not None.

66
tuskar/tests/manager/test_plan.py
View File

@ -25,6 +25,7 @@ from tuskar.storage.stores import DeploymentPlanStore
from tuskar.storage.stores import EnvironmentFileStore
from tuskar.storage.stores import MasterSeedStore
from tuskar.storage.stores import MasterTemplateStore
from tuskar.storage.stores import ResourceRegistryMappingStore
from tuskar.storage.stores import ResourceRegistryStore
from tuskar.storage.stores import TemplateStore
from tuskar.templates import namespace as ns_utils
@ -109,6 +110,7 @@ resources:
RESOURCE_REGISTRY = """
resource_registry:
OS::TripleO::Role: r1.yaml
OS::TripleO::Another: required_file.yaml
"""
RESOURCE_REGISTRY_WRONG_TYPE = """
@ -127,6 +129,7 @@ class PlansManagerTestCase(TestCase):
self.template_store = TemplateStore()
self.seed_store = MasterSeedStore()
self.registry_store = ResourceRegistryStore()
self.registry_mapping_store = ResourceRegistryMappingStore()
def test_create_plan(self):
# Tests
@ -182,6 +185,9 @@ class PlansManagerTestCase(TestCase):
# Setup
self.seed_store.create(MASTER_SEED_NAME, TEST_SEED)
self.registry_store.create(RESOURCE_REGISTRY_NAME, RESOURCE_REGISTRY)
# more setup (this is normally called in load_roles)
self.registry_mapping_store.create('required_file.yaml',
'some fake template data')
test_role = self._add_test_role()
test_plan = self.plans_manager.create_plan('p1', 'd1')
@ -206,6 +212,12 @@ class PlansManagerTestCase(TestCase):
{'ip_addresses':
{'get_attr': ['r1-1-servers', 'foo_ip']}})
# verify both entries are present from RESOURCE_REGISTRY
parsed_env = parser.parse_environment(
db_plan.environment_file.contents
)
self.assertEqual(2, len(parsed_env.registry_entries))
def test_add_unknown_role_to_seeded_plan(self):
# Setup
self.seed_store.create(MASTER_SEED_NAME, TEST_SEED)
@ -388,5 +400,59 @@ class PlansManagerTestCase(TestCase):
parsed_role = parser.parse_template(templates[role_filename])
self.assertEqual(parsed_role.description, 'Test provider resource foo')
def test_package_templates_seeded_plan(self):
# Setup
self.seed_store.create(MASTER_SEED_NAME, TEST_SEED)
self.registry_store.create(RESOURCE_REGISTRY_NAME, RESOURCE_REGISTRY)
# more setup (this is normally called in load_roles)
self.registry_mapping_store.create('required_file.yaml',
'some fake template data')
test_role = self._add_test_role()
test_plan = self.plans_manager.create_plan('p1', 'd1')
self.plans_manager.add_role_to_plan(test_plan.uuid, test_role.uuid)
# Test
templates = self.plans_manager.package_templates(test_plan.uuid)
# Verify
self.assertTrue(isinstance(templates, dict))
self.assertEqual(4, len(templates))
self.assertTrue('plan.yaml' in templates)
parsed_plan = parser.parse_template(templates['plan.yaml'])
self.assertEqual(parsed_plan.description, 'd1')
self.assertTrue('environment.yaml' in templates)
self.assertTrue('required_file.yaml' in templates)
parsed_env = parser.parse_environment(templates['environment.yaml'])
self.assertEqual(2, len(parsed_env.registry_entries))
role_filename = name_utils.role_template_filename('r1', '1')
self.assertTrue(role_filename in templates)
parsed_role = parser.parse_template(templates[role_filename])
self.assertEqual(parsed_role.description, 'Test provider resource foo')
def test_find_roles(self):
# Setup
self.seed_store.create(MASTER_SEED_NAME, TEST_SEED)
self.registry_store.create(RESOURCE_REGISTRY_NAME, RESOURCE_REGISTRY)
# more setup (this is normally called in load_roles)
self.registry_mapping_store.create('required_file.yaml',
'some fake template data')
test_role = self._add_test_role()
test_plan = self.plans_manager.create_plan('p1', 'd1')
# Test
self.plans_manager.add_role_to_plan(test_plan.uuid, test_role.uuid)
# Verify only one role is found
db_plan = self.plan_store.retrieve(test_plan.uuid)
parsed_env = parser.parse_environment(
db_plan.environment_file.contents
)
roles = self.plans_manager._find_roles(parsed_env)
self.assertEqual(1, len(roles))
def _add_test_role(self):
return self.template_store.create('r1', TEST_TEMPLATE)

44
tuskar/tests/storage/test_load_roles.py
View File

@ -33,19 +33,22 @@ class LoadRoleTests(TestCase):
self.roles = [path.join(self.directory, role) for role in roles_name]
for role in self.roles:
self._create_role(role)
self._create_file(role)
def tearDown(self):
super(LoadRoleTests, self).tearDown()
rmtree(self.directory)
def _create_role(self, role):
"""Create a mock role file which simple contains it's own name as
the file contents.
def _create_file(self, file, data=None):
"""Create a mock file which contains its own name as the file
contents when the data argument is empty.
"""
with open(role, 'w') as f:
f.write("CONTENTS FOR {0}".format(role))
if data is None:
data = "CONTENTS FOR {0}".format(file)
with open(file, 'w') as f:
f.write(data)
def test_dry_run(self):
@ -83,7 +86,7 @@ class LoadRoleTests(TestCase):
def test_import_with_seed(self):
# Setup
self._create_role(path.join(self.directory, 'seed'))
self._create_file(path.join(self.directory, 'seed'))
# Test
seed_file = path.join(self.directory, 'seed')
@ -94,3 +97,30 @@ class LoadRoleTests(TestCase):
self.assertEqual(['_master_seed', 'role1', 'role2'], sorted(total))
self.assertEqual(['_master_seed', 'role1', 'role2'], sorted(created))
self.assertEqual([], updated)
def test_import_seed_and_registry(self):
env_data = """
resource_registry:
OS::TripleO::Role: role1.yaml
OS::TripleO::Another: required_file.yaml
"""
# Setup
self._create_file(path.join(self.directory, 'seed'))
self._create_file(path.join(self.directory, 'environment'), env_data)
self._create_file(path.join(self.directory, 'required_file.yaml'))
# Test
seed_file = path.join(self.directory, 'seed')
env_file = path.join(self.directory, 'environment')
all_roles, created, updated = load_roles.load_roles(
self.roles,
seed_file=seed_file,
resource_registry_path=env_file)
# Verify
self.assertEqual(['_master_seed', '_registry',
'role1', 'role2'], sorted(all_roles))
self.assertEqual(['_master_seed', '_registry', 'required_file.yaml',
'role1', 'role2'], sorted(created))
self.assertEqual([], updated)

16
tuskar/tests/templates/test_heat.py
View File

@ -433,6 +433,11 @@ class EnvironmentTests(unittest.TestCase):
e.remove_registry_entry(re)
self.assertEqual(0, len(e.registry_entries))
# Test unique add
e.add_registry_entry(re, unique=True)
e.add_registry_entry(re, unique=True)
self.assertEqual(1, len(e.registry_entries))
def test_remove_registry_entry_not_found(self):
e = heat.Environment()
self.assertRaises(ValueError, e.remove_registry_entry,
@ -495,6 +500,17 @@ class EnvironmentParameterTests(unittest.TestCase):
self.assertEqual('test-value', p.value)
class RegistryEntryTest(unittest.TestCase):
def test_is_filename(self):
re = heat.RegistryEntry('Tuskar::compute-1', 'provider-compute-1.yaml')
self.assertTrue(re.is_filename())
re = heat.RegistryEntry('OS::TripleO::StructuredDeployment',
'OS::Heat::StructuredDeployment')
self.assertFalse(re.is_filename())
class ModuleMethodTests(unittest.TestCase):
def test_safe_strip(self):