From 4451a173fa5ba4c47f04395d0b65e7e014dbe147 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 13 Feb 2015 16:21:05 +0200 Subject: [PATCH] Make one joined graph from 3 separate stages Previously this 3 stages was disjoined and connected programatically during traversal in nailgun. Such approach imposes some dificulties during traversal and filtering and not really extandable (add new stages). Introduced change allows us to simplify traversal logic, and therefore improve UX for tasks developer Also in such approach it is quite easy to introduce new stages, like something that should be executed after cluster is completed, for example, stage end_notification that should fire some messages when we can guarantee that cluster is completed. Or add stage provision to the same graph. related to blueprint granular-deployment-based-on-tasks Depends on nailgun: Ida5880525f9c3428ad8bb590f7ef78a0b20b1049 Change-Id: Ie332d0c1745a48c5ae96c4c80d94922c321d5f7a --- .../puppet/deployment_groups/tasks.yaml | 141 ++++++++++-------- .../osnailyfacter/modular/astute/tasks.yaml | 46 ++++-- .../modular/cluster-haproxy/tasks.yaml | 2 +- .../osnailyfacter/modular/cluster/tasks.yaml | 2 +- .../modular/openstack-controller/tasks.yaml | 2 +- .../modular/openstack-haproxy/tasks.yaml | 2 +- .../puppet/osnailyfacter/modular/tasks.yaml | 29 ++-- .../modular/virtual_ips/tasks.yaml | 2 +- .../osnailyfacter/modular/vmware/tasks.yaml | 4 +- .../test_configuration.py | 2 - 10 files changed, 133 insertions(+), 99 deletions(-) diff --git a/deployment/puppet/deployment_groups/tasks.yaml b/deployment/puppet/deployment_groups/tasks.yaml index 7f3610f0e7..16f7b5b9d4 100644 --- a/deployment/puppet/deployment_groups/tasks.yaml +++ b/deployment/puppet/deployment_groups/tasks.yaml @@ -1,91 +1,108 @@ #STAGES -- id: deploy + +- id: pre_deployment_start type: stage -- id: pre_deployment + +- id: pre_deployment_end type: stage -- id: post_deployment + requires: [pre_deployment_start] + +- id: deploy_start type: stage + requires: [pre_deployment_end] + +- id: deploy_end + type: stage + requires: [deploy_start] + +- id: post_deployment_start + type: stage + requires: [deploy_end] + +- id: post_deployment_end + type: stage + requires: [post_deployment_start] #GROUPS - id: primary-controller type: group role: [primary-controller] - required_for: [deploy] + requires: [deploy_start] + required_for: [deploy_end] parameters: strategy: type: one_by_one + - id: controller type: group role: [controller] requires: [primary-controller] - required_for: [deploy] + required_for: [deploy_end] parameters: strategy: type: parallel amount: 6 + - id: cinder type: group role: [cinder] requires: [controller] - required_for: [deploy] - parameters: - strategy: - type: parallel -- id: compute - type: group - role: [compute] - requires: [controller] - required_for: [deploy] - parameters: - strategy: - type: parallel -- id: zabbix-server - type: group - role: [zabbix-server] - required_for: [deploy] - parameters: - strategy: - type: one_by_one -- id: mongo - type: group - role: [mongo] - requires: [zabbix-server] - required_for: [deploy, primary-controller, controller] - parameters: - strategy: - type: parallel -- id: primary-mongo - type: group - role: [primary-mongo] - requires: [mongo] - required_for: [deploy, primary-controller, controller] - parameters: - strategy: - type: one_by_one -- id: ceph-osd - type: group - role: [ceph-osd] - requires: [controller] - required_for: [deploy] - parameters: - strategy: - type: parallel -- id: base-os - type: group - role: [base-os] - tasks: [hiera, globals] - required_for: [deploy] + required_for: [deploy_end] parameters: strategy: type: parallel -#PRE DEPLOYMENT -- id: sync_time - type: shell - role: '*' - stage: pre_deployment +- id: compute + type: group + role: [compute] + requires: [controller] + required_for: [deploy_end] parameters: - cmd: ntpdate -u $(egrep '^server' /etc/ntp.conf | egrep -v '127\.127\.[0-9]+\.[0-9]+' | sed '/^#/d' | awk '{print $2}') - retries: 10 - interval: 1 - timeout: 180 + strategy: + type: parallel + +- id: zabbix-server + type: group + role: [zabbix-server] + required_for: [deploy_end] + requires: [deploy_start] + parameters: + strategy: + type: one_by_one + +- id: mongo + type: group + role: [mongo] + requires: [zabbix-server] + required_for: [deploy_end, primary-controller, controller] + parameters: + strategy: + type: parallel + +- id: primary-mongo + type: group + role: [primary-mongo] + requires: [mongo] + required_for: [deploy_end, primary-controller, controller] + parameters: + strategy: + type: one_by_one + +- id: ceph-osd + type: group + role: [ceph-osd] + requires: [controller] + required_for: [deploy_end] + parameters: + strategy: + type: parallel + +- id: base-os + type: group + role: [base-os] + tasks: [hiera, globals] + required_for: [deploy_end] + requires: [deploy_start] + parameters: + strategy: + type: parallel diff --git a/deployment/puppet/osnailyfacter/modular/astute/tasks.yaml b/deployment/puppet/osnailyfacter/modular/astute/tasks.yaml index 27d86220cb..3ed4f411dc 100644 --- a/deployment/puppet/osnailyfacter/modular/astute/tasks.yaml +++ b/deployment/puppet/osnailyfacter/modular/astute/tasks.yaml @@ -1,7 +1,9 @@ +# POST_DEPLOYMENT TASKS - id: enable_quorum type: shell role: [primary-controller] - stage: post_deployment + requires: [post_deployment_start] + required_for: [post_deployment_end] parameters: cmd: ruby /etc/puppet/modules/osnailyfacter/modular/astute/enable_quorum.rb timeout: 180 @@ -9,9 +11,9 @@ - id: restart_rados type: shell role: [primary-controller, controller] - stage: post_deployment condition: "settings:storage.objects_ceph.value == true" - required_for: [upload_cirros] + requires: [post_deployment_start] + required_for: [upload_cirros, post_deployment_end] parameters: cmd: sh /etc/puppet/modules/osnailyfacter/modular/astute/restart_rados.sh timeout: 180 @@ -20,7 +22,7 @@ type: shell role: [primary-controller] requires: [enable_quorum] - stage: post_deployment + required_for: [post_deployment_end] parameters: cmd: ruby /etc/puppet/modules/osnailyfacter/modular/astute/upload_cirros.rb timeout: 180 @@ -28,14 +30,14 @@ - id: upload_nodes_info type: upload_file role: '*' - stage: post_deployment + requires: [post_deployment_start] parameters: path: /etc/hiera/nodes.yaml - id: update_hosts type: puppet role: '*' - stage: post_deployment + required_for: [post_deployment_end] requires: [upload_nodes_info] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/hosts.pp @@ -47,41 +49,45 @@ - id: vcenter_compute_zones_create type: shell role: [primary-controller] - stage: post_deployment + requires: [post_deployment_start] + required_for: [post_deployment_end] condition: "settings:common.use_vcenter.value == true" parameters: cmd: /usr/bin/python /etc/puppet/modules/osnailyfacter/modular/astute/vcenter_compute_zones.py timeout: 180 +#PRE DEPLOYMENT - id: upload_core_repos type: upload_file role: '*' - stage: pre_deployment + requires: [pre_deployment_start] - id: rsync_core_puppet type: sync role: '*' - stage: pre_deployment + required_for: [pre_deployment_end] requires: [upload_core_repos] parameters: src: rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/ dst: /etc/puppet - timeout: + timeout: 180 + - id: clear_nodes_info type: shell role: '*' - stage: pre_deployment + requires: [pre_deployment_start] + required_for: [pre_deployment_end] parameters: cmd: rm -f /etc/hiera/nodes.yaml retries: 1 -#PREDEPLOYMENT HOOKS + - id: copy_keys type: copy_files role: '*' - stage: pre_deployment + required_for: [pre_deployment_end] requires: [generate_keys] parameters: files: @@ -109,8 +115,20 @@ - id: generate_keys type: shell role: master - stage: pre_deployment + requires: [pre_deployment_start] required_for: [copy_keys] parameters: cmd: sh /etc/puppet/modules/osnailyfacter/modular/astute/generate_keys.sh -i {CLUSTER_ID} -o 'mongodb' -s 'neutron nova ceph mysql' -p /var/lib/fuel/keys/ timeout: 180 + +- id: sync_time + type: shell + role: '*' + requires: [pre_deployment_start] + required_for: [pre_deployment_end] + parameters: + cmd: ntpdate -u $(egrep '^server' /etc/ntp.conf | egrep -v '127\.127\.[0-9]+\.[0-9]+' | sed '/^#/d' | awk '{print $2}') + retries: 10 + interval: 1 + timeout: 180 + diff --git a/deployment/puppet/osnailyfacter/modular/cluster-haproxy/tasks.yaml b/deployment/puppet/osnailyfacter/modular/cluster-haproxy/tasks.yaml index babe2fb6d7..fc8c550d46 100644 --- a/deployment/puppet/osnailyfacter/modular/cluster-haproxy/tasks.yaml +++ b/deployment/puppet/osnailyfacter/modular/cluster-haproxy/tasks.yaml @@ -1,7 +1,7 @@ - id: cluster-haproxy type: puppet groups: [primary-controller, controller] - required_for: [top-role-controller, deploy] + required_for: [top-role-controller, deploy_end] requires: [virtual_ips] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/cluster-haproxy/cluster-haproxy.pp diff --git a/deployment/puppet/osnailyfacter/modular/cluster/tasks.yaml b/deployment/puppet/osnailyfacter/modular/cluster/tasks.yaml index bc9edd5636..143d1e2865 100644 --- a/deployment/puppet/osnailyfacter/modular/cluster/tasks.yaml +++ b/deployment/puppet/osnailyfacter/modular/cluster/tasks.yaml @@ -1,7 +1,7 @@ - id: cluster type: puppet groups: [primary-controller, controller] - required_for: [top-role-controller, deploy] + required_for: [top-role-controller, deploy_end] requires: [hosts, firewall] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/cluster/cluster.pp diff --git a/deployment/puppet/osnailyfacter/modular/openstack-controller/tasks.yaml b/deployment/puppet/osnailyfacter/modular/openstack-controller/tasks.yaml index ec9990a93c..eb0629d519 100644 --- a/deployment/puppet/osnailyfacter/modular/openstack-controller/tasks.yaml +++ b/deployment/puppet/osnailyfacter/modular/openstack-controller/tasks.yaml @@ -1,7 +1,7 @@ - id: openstack-controller type: puppet groups: [primary-controller, controller] - required_for: [top-role-controller, deploy] + required_for: [top-role-controller, deploy_end] requires: [openstack-haproxy] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/openstack-controller/openstack-controller.pp diff --git a/deployment/puppet/osnailyfacter/modular/openstack-haproxy/tasks.yaml b/deployment/puppet/osnailyfacter/modular/openstack-haproxy/tasks.yaml index adbf752379..7c48b43920 100644 --- a/deployment/puppet/osnailyfacter/modular/openstack-haproxy/tasks.yaml +++ b/deployment/puppet/osnailyfacter/modular/openstack-haproxy/tasks.yaml @@ -1,7 +1,7 @@ - id: openstack-haproxy type: puppet groups: [primary-controller, controller] - required_for: [top-role-controller, deploy] + required_for: [top-role-controller, deploy_end] requires: [cluster-haproxy] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/openstack-haproxy/openstack-haproxy.pp diff --git a/deployment/puppet/osnailyfacter/modular/tasks.yaml b/deployment/puppet/osnailyfacter/modular/tasks.yaml index 8c1c76809e..5ddb108d8c 100644 --- a/deployment/puppet/osnailyfacter/modular/tasks.yaml +++ b/deployment/puppet/osnailyfacter/modular/tasks.yaml @@ -1,7 +1,7 @@ - id: netconfig type: puppet groups: [primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo] - required_for: [deploy] + required_for: [deploy_end] requires: [logging] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/netconfig.pp @@ -11,7 +11,7 @@ - id: logging type: puppet groups: [primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo] - required_for: [deploy] + required_for: [deploy_end] requires: [globals] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/logging.pp @@ -21,7 +21,7 @@ - id: tools type: puppet groups: [primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo] - required_for: [deploy] + required_for: [deploy_end] requires: [logging] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/tools.pp @@ -31,7 +31,7 @@ - id: hosts type: puppet groups: [primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo] - required_for: [deploy] + required_for: [deploy_end] requires: [netconfig] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/hosts.pp @@ -41,7 +41,7 @@ - id: firewall type: puppet groups: [primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo] - required_for: [deploy] + required_for: [deploy_end] requires: [netconfig] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/firewall.pp @@ -51,7 +51,8 @@ - id: hiera type: puppet groups: [primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo] - required_for: [deploy] + requires: [deploy_start] + required_for: [deploy_end] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/hiera.pp puppet_modules: /etc/puppet/modules @@ -62,7 +63,7 @@ groups: [primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo] - required_for: [deploy] + required_for: [deploy_end] requires: [hiera] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/globals.pp @@ -72,7 +73,7 @@ - id: top-role-controller type: puppet groups: [primary-controller, controller] - required_for: [deploy] + required_for: [deploy_end] requires: [hosts, firewall] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/controller.pp @@ -82,7 +83,7 @@ - id: top-role-compute type: puppet groups: [compute] - required_for: [deploy] + required_for: [deploy_end] requires: [hosts, firewall] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/compute.pp @@ -92,7 +93,7 @@ - id: top-role-cinder type: puppet groups: [cinder] - required_for: [deploy] + required_for: [deploy_end] requires: [hosts, firewall] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/cinder.pp @@ -102,7 +103,7 @@ - id: top-role-ceph-osd type: puppet groups: [ceph-osd] - required_for: [deploy] + required_for: [deploy_end] requires: [hosts, firewall] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/ceph-osd.pp @@ -112,7 +113,7 @@ - id: top-role-primary-mongo type: puppet groups: [primary-mongo] - required_for: [deploy] + required_for: [deploy_end] requires: [hosts, firewall] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/mongo_primary.pp @@ -122,7 +123,7 @@ - id: top-role-mongo type: puppet groups: [mongo] - required_for: [deploy] + required_for: [deploy_end] requires: [hosts, firewall] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/mongo.pp @@ -134,7 +135,7 @@ groups: [primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo] - required_for: [deploy] + required_for: [deploy_end] requires: [hosts, firewall] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/zabbix.pp diff --git a/deployment/puppet/osnailyfacter/modular/virtual_ips/tasks.yaml b/deployment/puppet/osnailyfacter/modular/virtual_ips/tasks.yaml index 36b159c713..07b716e878 100644 --- a/deployment/puppet/osnailyfacter/modular/virtual_ips/tasks.yaml +++ b/deployment/puppet/osnailyfacter/modular/virtual_ips/tasks.yaml @@ -1,7 +1,7 @@ - id: virtual_ips type: puppet groups: [primary-controller, controller] - required_for: [top-role-controller, deploy] + required_for: [top-role-controller, deploy_end] requires: [cluster] parameters: puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/virtual_ips/virtual_ips.pp diff --git a/deployment/puppet/osnailyfacter/modular/vmware/tasks.yaml b/deployment/puppet/osnailyfacter/modular/vmware/tasks.yaml index a98edc055f..029b484dfd 100644 --- a/deployment/puppet/osnailyfacter/modular/vmware/tasks.yaml +++ b/deployment/puppet/osnailyfacter/modular/vmware/tasks.yaml @@ -1,7 +1,7 @@ - id: vmware-vcenter type: puppet groups: [primary-controller, controller] - required_for: [deploy] + required_for: [deploy_end] requires: [top-role-controller] condition: "settings:common.libvirt_type.value == 'vcenter' or settings:common.use_vcenter.value == true" parameters: @@ -12,7 +12,7 @@ - id: vmware-compute type: puppet groups: [compute] - required_for: [deploy] + required_for: [deploy_end] requires: [top-role-compute] condition: "settings:common.libvirt_type.value == 'vcenter' or settings:common.use_vcenter.value == true" parameters: diff --git a/utils/configuration_validator/test_configuration.py b/utils/configuration_validator/test_configuration.py index 41b6cd496d..5f0d69a719 100644 --- a/utils/configuration_validator/test_configuration.py +++ b/utils/configuration_validator/test_configuration.py @@ -82,8 +82,6 @@ def graph(tasks): for req in task['tasks']: graph.add_edge(req, task['id']) - if 'stage' in task: - graph.add_edge(task['id'], task['stage']) return graph