openstack
/
fuel-web
Code Issues Proposed changes

Updated JSON schema for task 

- Moved all Task Role related constants to Enum TASK_ROLES
- Renamed consts.MASTER_ROLE to consts.MASTER_NODE_UID

Change-Id: I292f0f5471e295564a318530f5397e4c438cb40a
Closed-Bug: #1526688
This commit is contained in:
Bulat Gaifullin 2015-12-16 14:17:51 +03:00
parent 26c78e1c29
commit 7a4d5de086
14 changed files with 165 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
nailgun/nailgun/api/v1/validators/json_schema/tasks.py
View File

@ -13,7 +13,27 @@
# License for the specific language governing permissions and limitations
# under the License.
from nailgun.consts import NODE_RESOLVE_POLICY
from nailgun.consts import ORCHESTRATOR_TASK_TYPES
from nailgun.consts import TASK_ROLES
RELATION_SCHEMA = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'type': 'object',
'required': ['name'],
'properties': {
'name': {'type': 'string'},
'role': {
'oneOf': [
{'type': 'string', 'enum': list(TASK_ROLES)},
{'type': 'array'},
]
},
'policy': {'type': 'string', 'enum': list(NODE_RESOLVE_POLICY)},
}
}
TASK_SCHEMA = {
'$schema': 'http://json-schema.org/draft-04/schema#',
@ -23,9 +43,23 @@ TASK_SCHEMA = {
'id': {'type': 'string'},
'type': {'enum': list(ORCHESTRATOR_TASK_TYPES),
'type': 'string'},
'version': {'type': 'string', "pattern": "^\d+.\d+.\d+$"},
'parameters': {'type': 'object'},
'required_for': {'type': 'array'},
'requires': {'type': 'array'}}}
'requires': {'type': 'array'},
'cross-depends': {'type': 'array', 'items': RELATION_SCHEMA},
'cross-depended-by': {'type': 'array', 'items': RELATION_SCHEMA},
'strategy': {
'type': 'object',
'required': ['type'],
'properties': {
'type': {'enum': ['parallel', 'one-by-one'],
'type': 'string'},
'amount': {'type': 'integer'}
}
}
}
}
TASKS_SCHEMA = {

9
nailgun/nailgun/consts.py
View File

@ -375,9 +375,7 @@ TASK_REFRESH_FIELD = 'refresh_on'
ROLE_NAME_MAX_SIZE = 64
EXTENSION_NAME_MAX_SIZE = 64
ALL_ROLES = '*'
MASTER_ROLE = 'master'
ROLE_SELF_NODE = 'self'
MASTER_NODE_UID = 'master'
# version of Fuel when we added granular deploy support
FUEL_GRANULAR_DEPLOY = '6.1'
@ -446,6 +444,11 @@ NODE_RESOLVE_POLICY = Enum(
"any"
)
TASK_ROLES = Enum(
'*', 'master', 'self',
names=('all', 'master', 'self')
)
OVERRIDE_CONFIG_BASE_PATH = '/etc/hiera/override/configuration/'
TASK_CROSS_DEPENDENCY = '2.0.0'

4
nailgun/nailgun/orchestrator/plugins_serializers.py
View File

@ -143,8 +143,8 @@ class PluginsPreDeploymentHooksSerializer(BasePluginDeploymentHooksSerializer):
# commands need to be run are not compatible with master node
# OS (CentOS). E.g. of such situation - create repository
# executes `apt-get update` which fails on CentOS
if consts.MASTER_ROLE in uids:
uids.remove(consts.MASTER_ROLE)
if consts.MASTER_NODE_UID in uids:
uids.remove(consts.MASTER_NODE_UID)
return uids

8
nailgun/nailgun/orchestrator/provisioning_serializers.py
View File

@ -301,7 +301,7 @@ class ProvisioningSerializer61(ProvisioningSerializer):
if is_build_images:
tasks.append(
tasks_templates.make_provisioning_images_task(
[consts.MASTER_ROLE],
[consts.MASTER_NODE_UID],
attrs['repo_setup']['repos'],
attrs['provision'],
cluster.id))
@ -318,7 +318,7 @@ class ProvisioningSerializer61(ProvisioningSerializer):
if is_download_debian_installer:
tasks.append(
tasks_templates.make_download_debian_installer_task(
[consts.MASTER_ROLE],
[consts.MASTER_NODE_UID],
attrs['repo_setup']['repos'],
attrs['repo_setup']['installer_kernel'],
attrs['repo_setup']['installer_initrd']))
@ -385,12 +385,12 @@ class ProvisioningSerializer80(ProvisioningSerializer70):
if attrs['ironic']['enabled']:
tasks.append(
tasks_templates.generate_ironic_bootstrap_keys_task(
[consts.MASTER_ROLE],
[consts.MASTER_NODE_UID],
cluster.id))
tasks.append(
tasks_templates.make_ironic_bootstrap_task(
[consts.MASTER_ROLE],
[consts.MASTER_NODE_UID],
cluster.id))
PriorityStrategy().one_by_one(tasks)

4
nailgun/nailgun/orchestrator/task_based_deploy.py
View File

@ -448,9 +448,9 @@ class TasksSerializer(object):
return
for dep in dependencies:
roles = dep.get('role', consts.ALL_ROLES)
roles = dep.get('role', consts.TASK_ROLES.all)
if roles == consts.ROLE_SELF_NODE:
if roles == consts.TASK_ROLES.self:
node_ids = [node_id]
else:
node_ids = self.role_resolver.resolve(

20
nailgun/nailgun/orchestrator/tasks_serializer.py
View File

@ -43,10 +43,10 @@ def get_uids_for_tasks(nodes, tasks):
# plugin tasks may store information about node
# role not only in `role` key but also in `groups`
task_role = task.get('role', task.get('groups'))
if task_role == consts.ALL_ROLES:
return get_uids_for_roles(nodes, consts.ALL_ROLES)
elif task_role == consts.MASTER_ROLE:
return [consts.MASTER_ROLE]
if task_role == consts.TASK_ROLES.all:
return get_uids_for_roles(nodes, consts.TASK_ROLES.all)
elif task_role == consts.TASK_ROLES.master:
return [consts.MASTER_NODE_UID]
elif isinstance(task_role, list):
roles.extend(task_role)
# if task has 'skipped' status it is allowed that 'roles' and
@ -64,16 +64,16 @@ def get_uids_for_roles(nodes, roles):
"""Returns list of uids for nodes that matches roles
:param nodes: list of nodes
:param roles: list of roles or consts.ALL_ROLES
:param roles: list of roles or consts.TASK_ROLES.all
:returns: list of strings
"""
uids = set()
if roles == consts.ALL_ROLES:
if roles == consts.TASK_ROLES.all:
uids.update([n.uid for n in nodes])
elif roles == consts.MASTER_ROLE:
return [consts.MASTER_ROLE]
elif roles == consts.TASK_ROLES.master:
return [consts.MASTER_NODE_UID]
elif isinstance(roles, list):
for node in nodes:
if set(roles) & set(objects.Node.all_roles(node)):
@ -180,7 +180,7 @@ class UploadMOSRepo(GenericRolesHook):
identity = 'upload_core_repos'
def get_uids(self):
return self.role_resolver.resolve(consts.ALL_ROLES)
return self.role_resolver.resolve(consts.TASK_ROLES.all)
def serialize(self):
uids = self.get_uids()
@ -224,7 +224,7 @@ class RsyncPuppet(GenericRolesHook):
identity = 'rsync_core_puppet'
def get_uids(self):
return self.role_resolver.resolve(consts.ALL_ROLES)
return self.role_resolver.resolve(consts.TASK_ROLES.all)
def serialize(self):
src_path = self.task['parameters']['src'].format(

4
nailgun/nailgun/rpc/receiver.py
View File

@ -254,7 +254,7 @@ class NailgunReceiver(object):
# for deployment we need just to pop
master = next((
n for n in nodes if n['uid'] == consts.MASTER_ROLE), {})
n for n in nodes if n['uid'] == consts.MASTER_NODE_UID), {})
# we should remove master node from the nodes since it requires
# special handling and won't work with old code
@ -356,7 +356,7 @@ class NailgunReceiver(object):
# if task was failed on master node then we should
# mark all cluster's nodes in error state
master = next((
n for n in nodes if n['uid'] == consts.MASTER_ROLE), {})
n for n in nodes if n['uid'] == consts.MASTER_NODE_UID), {})
# we should remove master node from the nodes since it requires
# special handling and won't work with old code

8
nailgun/nailgun/task/task.py
View File

@ -607,9 +607,9 @@ class DeleteIBPImagesTask(object):
'execute_tasks',
'remove_images_resp',
{
'tasks': [tasks_templates.make_shell_task([consts.MASTER_ROLE],
task_params),
]
'tasks': [tasks_templates.make_shell_task(
[consts.MASTER_NODE_UID], task_params
)]
}
)
return rpc_message
@ -721,7 +721,7 @@ class RemoveClusterKeys(object):
{
"tasks": [
tasks_templates.make_shell_task(
[consts.MASTER_ROLE],
[consts.MASTER_NODE_UID],
{
"parameters": {
"cmd": "rm -rf /var/lib/fuel/keys/{0}".format(

91
nailgun/nailgun/test/integration/test_graph_related_handlers.py
View File

@ -39,6 +39,7 @@ class BaseGraphTasksTests(BaseIntegrationTest):
requires: [pre_deployment_end]
- id: deploy_end
type: stage
version: 1.0.0
requires: [deploy_start]
- id: pre_deployment_start
type: stage
@ -75,6 +76,16 @@ class BaseGraphTasksTests(BaseIntegrationTest):
"""
return yaml.load(yaml_tasks)
def get_tasks_with_unsuported_role(self):
yaml_tasks = """
- id: test-controller
role: "test-controller"
parameters:
strategy:
type: one_by_one
"""
return yaml.load(yaml_tasks)
def get_tasks_with_cycles(self):
yaml_tasks = """
- id: test-controller-1
@ -98,6 +109,53 @@ class BaseGraphTasksTests(BaseIntegrationTest):
"""
return yaml.load(yaml_tasks)
def get_tasks_with_cross_dependencies(self):
yaml_tasks = """
- id: test-controller
type: group
version: 2.0.0
role: [test-controller]
cross-depends:
- name: test-compute
role: '*'
polcy: any
parameters:
strategy:
type: one_by_one
- id: test-compute
type: group
version: 2.0.0
role: [test-compute]
cross-depends:
- name: test-cinder
role: [test_cinder]
parameters:
strategy:
type: one_by_one
- id: test-cinder
type: group
version: 2.0.0
role: [test-cinder]
parameters:
strategy:
type: one_by_one
"""
return yaml.load(yaml_tasks)
def get_tasks_cross_dependencies_without_name(self):
yaml_tasks = """
- id: test-controller
type: group
role: [test-controller]
cross-depends:
- role: '*'
polcy: any
parameters:
strategy:
type: one_by_one
"""
return yaml.load(yaml_tasks)
class TestReleaseGraphHandler(BaseGraphTasksTests):
@ -249,6 +307,39 @@ class TestClusterGraphHandler(BaseGraphTasksTests):
"groups|tasks for [test-controller] because they don't exist in "
"the graph", resp.json_body['message'])
def test_upload_tasks_without_unsupported_role(self):
tasks = self.get_tasks_with_unsuported_role()
resp = self.app.put(
reverse('ClusterDeploymentTasksHandler',
kwargs={'obj_id': self.cluster.id}),
params=jsonutils.dumps(tasks),
headers=self.default_headers,
expect_errors=True
)
self.assertEqual(resp.status_code, 400)
def test_upload_tasks_with_cross_dependencies(self):
tasks = self.get_tasks_with_cross_dependencies()
resp = self.app.put(
reverse('ClusterDeploymentTasksHandler',
kwargs={'obj_id': self.cluster.id}),
params=jsonutils.dumps(tasks),
headers=self.default_headers,
)
cluster_tasks = objects.Cluster.get_deployment_tasks(self.cluster)
self.assertEqual(cluster_tasks, resp.json)
def test_upload_cross_dependencies_without_name(self):
tasks = self.get_tasks_cross_dependencies_without_name()
resp = self.app.put(
reverse('ClusterDeploymentTasksHandler',
kwargs={'obj_id': self.cluster.id}),
params=jsonutils.dumps(tasks),
headers=self.default_headers,
expect_errors=True
)
self.assertEqual(resp.status_code, 400)
def test_post_tasks(self):
resp = self.app.post(
reverse('ClusterDeploymentTasksHandler',

6
nailgun/nailgun/test/integration/test_orchestrator_serializer_70.py
View File

@ -912,7 +912,7 @@ class TestPluginDeploymentTasksInjection(base.BaseIntegrationTest):
{
'id': 'pre-depl-plugin-task',
'type': 'puppet',
'role': consts.MASTER_ROLE,
'role': consts.MASTER_NODE_UID,
'requires': ['pre_deployment_start'],
'required_for': ['pre_deployment_end'],
'parameters': {
@ -929,7 +929,7 @@ class TestPluginDeploymentTasksInjection(base.BaseIntegrationTest):
{
'id': 'pre-depl-plugin-task-for-master-and-contr',
'type': 'puppet',
'groups': [consts.MASTER_ROLE,
'groups': [consts.MASTER_NODE_UID,
'primary-controller'],
'requires': ['pre_deployment_start'],
'required_for': ['pre_deployment_end'],
@ -952,7 +952,7 @@ class TestPluginDeploymentTasksInjection(base.BaseIntegrationTest):
graph, self.cluster, self.cluster.nodes)
for st in pre_deployment:
self.assertNotIn(consts.MASTER_ROLE, st['uids'])
self.assertNotIn(consts.MASTER_NODE_UID, st['uids'])
def test_plugin_depl_task_in_post_depl(self):
self.prepare_plugins_for_cluster(

4
nailgun/nailgun/test/unit/test_role_resolver.py
View File

@ -67,8 +67,8 @@ class TestPatternBasedRoleResolver(BaseUnitTest):
def test_resolve_master(self):
resolver = role_resolver.RoleResolver(self.nodes)
self.assertEqual(
[consts.MASTER_ROLE],
resolver.resolve(consts.MASTER_ROLE)
[consts.MASTER_NODE_UID],
resolver.resolve(consts.TASK_ROLES.master)
)
def test_resolve_any(self):

2
nailgun/nailgun/test/unit/test_task.py
View File

@ -115,7 +115,7 @@ class TestDeleteIBPImagesTask(BaseTestCase):
self.assertEqual(rpc_message, {
'tasks': [{
'type': 'shell',
'uids': [consts.MASTER_ROLE],
'uids': [consts.MASTER_NODE_UID],
'parameters': {
'retries': 3,
'cwd': '/',

2
nailgun/nailgun/test/unit/test_task_based_deploy.py
View File

@ -207,7 +207,7 @@ class TestTaskSerializers(BaseTestCase):
)
)
m_resolve.resolve.assert_called_with(
consts.ALL_ROLES, consts.NODE_RESOLVE_POLICY.all
consts.TASK_ROLES.all, consts.NODE_RESOLVE_POLICY.all
)
# concrete role and policy
self.assertItemsEqual(

4
nailgun/nailgun/utils/role_resolver.py
View File

@ -65,7 +65,7 @@ class RoleResolver(BaseRoleResolver):
# the mapping roles, those are resolved to known list of IDs
# master is used to run tasks on master node
SPECIAL_ROLES = {
consts.MASTER_ROLE: [consts.MASTER_ROLE]
consts.TASK_ROLES.master: [consts.MASTER_NODE_UID]
}
def __init__(self, nodes):
@ -81,7 +81,7 @@ class RoleResolver(BaseRoleResolver):
def resolve(self, roles, policy=None):
if isinstance(roles, six.string_types) and roles in self.SPECIAL_ROLES:
result = self.SPECIAL_ROLES[roles]
elif roles == consts.ALL_ROLES:
elif roles == consts.TASK_ROLES.all:
result = list(set(
uid for nodes in six.itervalues(self.__mapping)
for uid in nodes