Make one joined graph from 3 separate stages

Previously this 3 stages was disjoined and connected programatically
during traversal in nailgun. Such approach imposes some dificulties
during traversal and filtering and not really extandable (add new stages).

Introduced change allows us to simplify traversal logic, and therefore
improve UX for tasks developer

Also in such approach it is quite easy to introduce new stages, like
something that should be executed after cluster is completed, for example,
stage end_notification that should fire some messages when we can guarantee
that cluster is completed.
Or add stage provision to the same graph.

related to blueprint granular-deployment-based-on-tasks
Depends on nailgun: Ida5880525f9c3428ad8bb590f7ef78a0b20b1049

Change-Id: Ie332d0c1745a48c5ae96c4c80d94922c321d5f7a

deployment/puppet/deployment_groups/tasks.yaml

@@ -1,91 +1,108 @@
 #STAGES
-- id: deploy
+
+- id: pre_deployment_start
   type: stage
-- id: pre_deployment
+
+- id: pre_deployment_end
   type: stage
-- id: post_deployment
+  requires: [pre_deployment_start]
+
+- id: deploy_start
   type: stage
+  requires: [pre_deployment_end]
+
+- id: deploy_end
+  type: stage
+  requires: [deploy_start]
+
+- id: post_deployment_start
+  type: stage
+  requires: [deploy_end]
+
+- id: post_deployment_end
+  type: stage
+  requires: [post_deployment_start]
 
 #GROUPS
 - id: primary-controller
   type: group
   role: [primary-controller]
-  required_for: [deploy]
+  requires: [deploy_start]
+  required_for: [deploy_end]
   parameters:
     strategy:
       type: one_by_one
+
 - id: controller
   type: group
   role: [controller]
   requires: [primary-controller]
-  required_for: [deploy]
+  required_for: [deploy_end]
   parameters:
     strategy:
       type: parallel
       amount: 6
+
 - id: cinder
   type: group
   role: [cinder]
   requires: [controller]
-  required_for: [deploy]
+  required_for: [deploy_end]
-  parameters:
-    strategy:
-      type: parallel
-- id: compute
-  type: group
-  role: [compute]
-  requires: [controller]
-  required_for: [deploy]
-  parameters:
-    strategy:
-      type: parallel
-- id: zabbix-server
-  type: group
-  role: [zabbix-server]
-  required_for: [deploy]
-  parameters:
-    strategy:
-      type: one_by_one
-- id: mongo
-  type: group
-  role: [mongo]
-  requires: [zabbix-server]
-  required_for: [deploy, primary-controller, controller]
-  parameters:
-    strategy:
-      type: parallel
-- id: primary-mongo
-  type: group
-  role: [primary-mongo]
-  requires: [mongo]
-  required_for: [deploy, primary-controller, controller]
-  parameters:
-    strategy:
-      type: one_by_one
-- id: ceph-osd
-  type: group
-  role: [ceph-osd]
-  requires: [controller]
-  required_for: [deploy]
-  parameters:
-    strategy:
-      type: parallel
-- id: base-os
-  type: group
-  role: [base-os]
-  tasks: [hiera, globals]
-  required_for: [deploy]
   parameters:
     strategy:
       type: parallel
 
-#PRE DEPLOYMENT
+- id: compute
-- id: sync_time
+  type: group
-  type: shell
+  role: [compute]
-  role: '*'
+  requires: [controller]
-  stage: pre_deployment
+  required_for: [deploy_end]
   parameters:
-    cmd: ntpdate -u $(egrep '^server' /etc/ntp.conf | egrep -v '127\.127\.[0-9]+\.[0-9]+' | sed '/^#/d' | awk '{print $2}')
+    strategy:
-    retries: 10
+      type: parallel
-    interval: 1
+
-    timeout: 180
+- id: zabbix-server
+  type: group
+  role: [zabbix-server]
+  required_for: [deploy_end]
+  requires: [deploy_start]
+  parameters:
+    strategy:
+      type: one_by_one
+
+- id: mongo
+  type: group
+  role: [mongo]
+  requires: [zabbix-server]
+  required_for: [deploy_end, primary-controller, controller]
+  parameters:
+    strategy:
+      type: parallel
+
+- id: primary-mongo
+  type: group
+  role: [primary-mongo]
+  requires: [mongo]
+  required_for: [deploy_end, primary-controller, controller]
+  parameters:
+    strategy:
+      type: one_by_one
+
+- id: ceph-osd
+  type: group
+  role: [ceph-osd]
+  requires: [controller]
+  required_for: [deploy_end]
+  parameters:
+    strategy:
+      type: parallel
+
+- id: base-os
+  type: group
+  role: [base-os]
+  tasks: [hiera, globals]
+  required_for: [deploy_end]
+  requires: [deploy_start]
+  parameters:
+    strategy:
+      type: parallel

deployment/puppet/osnailyfacter/modular/astute/tasks.yaml

@@ -1,7 +1,9 @@
+# POST_DEPLOYMENT TASKS
 - id: enable_quorum
   type: shell
   role: [primary-controller]
-  stage: post_deployment
+  requires: [post_deployment_start]
+  required_for: [post_deployment_end]
   parameters:
     cmd: ruby /etc/puppet/modules/osnailyfacter/modular/astute/enable_quorum.rb
     timeout: 180
@@ -9,9 +11,9 @@
 - id: restart_rados
   type: shell
   role: [primary-controller, controller]
-  stage: post_deployment
   condition: "settings:storage.objects_ceph.value == true"
-  required_for: [upload_cirros]
+  requires: [post_deployment_start]
+  required_for: [upload_cirros, post_deployment_end]
   parameters:
     cmd: sh /etc/puppet/modules/osnailyfacter/modular/astute/restart_rados.sh
     timeout: 180
@@ -20,7 +22,7 @@
   type: shell
   role: [primary-controller]
   requires: [enable_quorum]
-  stage: post_deployment
+  required_for: [post_deployment_end]
   parameters:
     cmd: ruby /etc/puppet/modules/osnailyfacter/modular/astute/upload_cirros.rb
     timeout: 180
@@ -28,14 +30,14 @@
 - id: upload_nodes_info
   type: upload_file
   role: '*'
-  stage: post_deployment
+  requires: [post_deployment_start]
   parameters:
     path: /etc/hiera/nodes.yaml
 
 - id: update_hosts
   type: puppet
   role: '*'
-  stage: post_deployment
+  required_for: [post_deployment_end]
   requires: [upload_nodes_info]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/hosts.pp
@@ -47,41 +49,45 @@
 - id: vcenter_compute_zones_create
   type: shell
   role: [primary-controller]
-  stage: post_deployment
+  requires: [post_deployment_start]
+  required_for: [post_deployment_end]
   condition: "settings:common.use_vcenter.value == true"
   parameters:
     cmd: /usr/bin/python /etc/puppet/modules/osnailyfacter/modular/astute/vcenter_compute_zones.py
     timeout: 180
 
 
+#PRE DEPLOYMENT
 - id: upload_core_repos
   type: upload_file
   role: '*'
-  stage: pre_deployment
+  requires: [pre_deployment_start]
 
 - id: rsync_core_puppet
   type: sync
   role: '*'
-  stage: pre_deployment
+  required_for: [pre_deployment_end]
   requires: [upload_core_repos]
   parameters:
     src: rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/
     dst: /etc/puppet
-    timeout:
+    timeout: 180
+
 
 - id: clear_nodes_info
   type: shell
   role: '*'
-  stage: pre_deployment
+  requires: [pre_deployment_start]
+  required_for: [pre_deployment_end]
   parameters:
     cmd: rm -f /etc/hiera/nodes.yaml
     retries: 1
 
-#PREDEPLOYMENT HOOKS
+
 - id: copy_keys
   type: copy_files
   role: '*'
-  stage: pre_deployment
+  required_for: [pre_deployment_end]
   requires: [generate_keys]
   parameters:
     files:
@@ -109,8 +115,20 @@
 - id: generate_keys
   type: shell
   role: master
-  stage: pre_deployment
+  requires: [pre_deployment_start]
   required_for: [copy_keys]
   parameters:
     cmd: sh /etc/puppet/modules/osnailyfacter/modular/astute/generate_keys.sh -i {CLUSTER_ID} -o 'mongodb' -s 'neutron nova ceph mysql' -p /var/lib/fuel/keys/
     timeout: 180
+
+- id: sync_time
+  type: shell
+  role: '*'
+  requires: [pre_deployment_start]
+  required_for: [pre_deployment_end]
+  parameters:
+    cmd: ntpdate -u $(egrep '^server' /etc/ntp.conf | egrep -v '127\.127\.[0-9]+\.[0-9]+' | sed '/^#/d' | awk '{print $2}')
+    retries: 10
+    interval: 1
+    timeout: 180
+

deployment/puppet/osnailyfacter/modular/cluster-haproxy/tasks.yaml

@@ -1,7 +1,7 @@
 - id: cluster-haproxy
   type: puppet
   groups: [primary-controller, controller]
-  required_for: [top-role-controller, deploy]
+  required_for: [top-role-controller, deploy_end]
   requires: [virtual_ips]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/cluster-haproxy/cluster-haproxy.pp

deployment/puppet/osnailyfacter/modular/cluster/tasks.yaml

@@ -1,7 +1,7 @@
 - id: cluster
   type: puppet
   groups: [primary-controller, controller]
-  required_for: [top-role-controller, deploy]
+  required_for: [top-role-controller, deploy_end]
   requires: [hosts, firewall]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/cluster/cluster.pp

deployment/puppet/osnailyfacter/modular/openstack-controller/tasks.yaml

@@ -1,7 +1,7 @@
 - id: openstack-controller
   type: puppet
   groups: [primary-controller, controller]
-  required_for: [top-role-controller, deploy]
+  required_for: [top-role-controller, deploy_end]
   requires: [openstack-haproxy]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/openstack-controller/openstack-controller.pp

deployment/puppet/osnailyfacter/modular/openstack-haproxy/tasks.yaml

@@ -1,7 +1,7 @@
 - id: openstack-haproxy
   type: puppet
   groups: [primary-controller, controller]
-  required_for: [top-role-controller, deploy]
+  required_for: [top-role-controller, deploy_end]
   requires: [cluster-haproxy]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/openstack-haproxy/openstack-haproxy.pp

deployment/puppet/osnailyfacter/modular/tasks.yaml

@@ -1,7 +1,7 @@
 - id: netconfig
   type: puppet
   groups: [primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo]
-  required_for: [deploy]
+  required_for: [deploy_end]
   requires: [logging]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/netconfig.pp
@@ -11,7 +11,7 @@
 - id: logging
   type: puppet
   groups: [primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo]
-  required_for: [deploy]
+  required_for: [deploy_end]
   requires: [globals]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/logging.pp
@@ -21,7 +21,7 @@
 - id: tools
   type: puppet
   groups: [primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo]
-  required_for: [deploy]
+  required_for: [deploy_end]
   requires: [logging]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/tools.pp
@@ -31,7 +31,7 @@
 - id: hosts
   type: puppet
   groups: [primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo]
-  required_for: [deploy]
+  required_for: [deploy_end]
   requires: [netconfig]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/hosts.pp
@@ -41,7 +41,7 @@
 - id: firewall
   type: puppet
   groups: [primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo]
-  required_for: [deploy]
+  required_for: [deploy_end]
   requires: [netconfig]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/firewall.pp
@@ -51,7 +51,8 @@
 - id: hiera
   type: puppet
   groups: [primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo]
-  required_for: [deploy]
+  requires: [deploy_start]
+  required_for: [deploy_end]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/hiera.pp
     puppet_modules: /etc/puppet/modules
@@ -62,7 +63,7 @@
   groups: [primary-controller, controller,
            cinder, compute, ceph-osd,
            zabbix-server, primary-mongo, mongo]
-  required_for: [deploy]
+  required_for: [deploy_end]
   requires: [hiera]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/globals.pp
@@ -72,7 +73,7 @@
 - id: top-role-controller
   type: puppet
   groups: [primary-controller, controller]
-  required_for: [deploy]
+  required_for: [deploy_end]
   requires: [hosts, firewall]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/controller.pp
@@ -82,7 +83,7 @@
 - id: top-role-compute
   type: puppet
   groups: [compute]
-  required_for: [deploy]
+  required_for: [deploy_end]
   requires: [hosts, firewall]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/compute.pp
@@ -92,7 +93,7 @@
 - id: top-role-cinder
   type: puppet
   groups: [cinder]
-  required_for: [deploy]
+  required_for: [deploy_end]
   requires: [hosts, firewall]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/cinder.pp
@@ -102,7 +103,7 @@
 - id: top-role-ceph-osd
   type: puppet
   groups: [ceph-osd]
-  required_for: [deploy]
+  required_for: [deploy_end]
   requires: [hosts, firewall]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/ceph-osd.pp
@@ -112,7 +113,7 @@
 - id: top-role-primary-mongo
   type: puppet
   groups: [primary-mongo]
-  required_for: [deploy]
+  required_for: [deploy_end]
   requires: [hosts, firewall]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/mongo_primary.pp
@@ -122,7 +123,7 @@
 - id: top-role-mongo
   type: puppet
   groups: [mongo]
-  required_for: [deploy]
+  required_for: [deploy_end]
   requires: [hosts, firewall]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/mongo.pp
@@ -134,7 +135,7 @@
   groups: [primary-controller, controller,
            cinder, compute, ceph-osd,
            zabbix-server, primary-mongo, mongo]
-  required_for: [deploy]
+  required_for: [deploy_end]
   requires: [hosts, firewall]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/zabbix.pp

deployment/puppet/osnailyfacter/modular/virtual_ips/tasks.yaml

@@ -1,7 +1,7 @@
 - id: virtual_ips
   type: puppet
   groups: [primary-controller, controller]
-  required_for: [top-role-controller, deploy]
+  required_for: [top-role-controller, deploy_end]
   requires: [cluster]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/virtual_ips/virtual_ips.pp

deployment/puppet/osnailyfacter/modular/vmware/tasks.yaml

@@ -1,7 +1,7 @@
 - id: vmware-vcenter
   type: puppet
   groups: [primary-controller, controller]
-  required_for: [deploy]
+  required_for: [deploy_end]
   requires: [top-role-controller]
   condition: "settings:common.libvirt_type.value == 'vcenter' or settings:common.use_vcenter.value == true"
   parameters:
@@ -12,7 +12,7 @@
 - id: vmware-compute
   type: puppet
   groups: [compute]
-  required_for: [deploy]
+  required_for: [deploy_end]
   requires: [top-role-compute]
   condition: "settings:common.libvirt_type.value == 'vcenter' or settings:common.use_vcenter.value == true"
   parameters:

utils/configuration_validator/test_configuration.py

@@ -82,8 +82,6 @@ def graph(tasks):
             for req in task['tasks']:
                 graph.add_edge(req, task['id'])
 
-        if 'stage' in task:
-            graph.add_edge(task['id'], task['stage'])
     return graph