Simplify signal/wait system to be more generic.

Scenario creators can now create a dependency between
nodes by adding fragments that specify a signal name
and which nodes need to wait for that signal. For an
example please see nodes/2_role.yaml

data/nodes/2_role.yaml

@@ -8,7 +8,7 @@ nodes:
 
     networks:
       ci:
-    flavor: 'm1.medium'
+    flavor: 'm1.tiny'
     image:  'precise-x86_64'
     fragments:
       - 'bash-top'
@@ -41,7 +41,7 @@ nodes:
       - 'puppet-master-fact'
       - 'puppet-setup'
       - 'puppet-site'
-      - 'build-sig-stack'
+      - 'SIGNAL_stack'
   control-server:
     vagrant_name: control_basevm
     memory:       3000
@@ -66,9 +66,9 @@ nodes:
       - 'puppet-modules'
       - 'hiera-config'
       - 'puppet-setup'
-      - 'stack-wait-build'
+      - 'WAIT_stack build-server'
       - 'puppet-agent'
-      - 'control-sig-compute'
+      - 'SIGNAL_control'
   compute-server02:
     networks:
       ci:
@@ -88,7 +88,7 @@ nodes:
       - 'puppet-modules'
       - 'hiera-config'
       - 'puppet-setup'
-      - 'stack-wait-build'
+      - 'WAIT_stack build-server'
       - 'puppet-agent'
-      - 'compute-wait-control'
+      - 'WAIT_control control-server'
       - 'puppet-agent'

stack-builder/build.py

@@ -199,15 +199,20 @@ def allocate_ports(q, network_id, test_id="", count=1):
 def metadata_update(scenario_yaml, ports):
     # IP addresses of particular nodes mapped to specified config
     # values to go into hiera + build scripts. See data/nodes/2_role.yaml
+    # all values will also be mapped to a generic key:
+    # meta['hostname_networkname'] = ip
     meta_update = {}
     for node, props in scenario_yaml['nodes'].items():
         for network, mappings in props['networks'].items():
             if mappings != None:
                 for mapping in mappings:
                     meta_update[mapping] = str(ports[node][network][0]['fixed_ips'][0]['ip_address'])
+            # replace dashes since bash variables can't contain dashes
+            # but be careful when using this since hostnames can't contain underscores
+            # so they need to be converted back
+            meta_update[network + '_' + node.replace('-', '_')] = str(ports[node][network][0]['fixed_ips'][0]['ip_address'])
     return meta_update
 
-
 # Not used atm
 def make_key(n, test_id):
     command = 'ssh-keygen -t rsa -q -N "" -f keys/'+test_id
@@ -266,7 +271,6 @@ def make(n, q, k, args):
             else:
                 ports[node][network] = allocate_ports(q, networks[network]['id'], test_id)
 
-
     dprint("networks")
     for net, value in networks.items():
         dprint (net + str(value))

stack-builder/fragment.py

@@ -4,6 +4,13 @@ import yaml
 
 from metadata import build_metadata
 
+def signal_type(fragment):
+    if fragment[:5] == 'WAIT_':
+        return 'wait'
+    if fragment[:7] == 'SIGNAL_':
+        return 'signal'
+    return None
+
 class PercentTemplate(string.Template):
     delimiter='%'
 
@@ -11,11 +18,36 @@ def available_fragments(d='./stack-builder/fragments'):
     return [d + '/' + f for f in os.listdir(d)]
 
 # Create a deploy script from a list of fragments
-def build_deploy(frag_list):
+def build_deploy(fragment_dir, frag_list, metadata):
     deploy_script = ""
     for f in frag_list:
-        with open(f, 'r') as frag:
-            deploy_script = deploy_script + frag.read() + '\n'
+        sig = signal_type(f)
+        if sig == 'wait':
+            # Wait syntax is as follows:
+            # WAIT_signal hostname1 hostnameN
+            with open(fragment_dir + '/WAIT_TEMPLATE', 'r') as temp:
+                spl = f.split(' ')
+                nodes = ""
+                for node in spl[1:]:
+                    # Handle debug case where there is none of this metadata
+                    if 'ci_'+node.replace('-', '_') not in metadata:
+                        metadata['ci_'+node.replace('-', '_')] = "{"+node+"_ip}"
+                        print "Fragment creation: Node " + node + " IP not present in metadata: using " + "{" + node + "_ip}"
+                    nodes = nodes + metadata['ci_'+node.replace('-', '_')] + " "
+
+                repl = { 'nodes': nodes, 'signal': spl[0].split('_')[1] }
+                frag = PercentTemplate(temp.read()).safe_substitute(repl)
+                deploy_script = deploy_script + frag + '\n'
+
+        elif sig == 'signal':
+            with open(fragment_dir + '/SIGNAL_TEMPLATE', 'r') as temp:
+                spl = f.split('_')
+                frag = PercentTemplate(temp.read()).safe_substitute({'signal': spl[1]})
+                deploy_script = deploy_script + frag + '\n'
+        else:
+            with open(fragment_dir + '/' + f, 'r') as frag:
+                deploy_script = deploy_script + frag.read() + '\n'
+
     return deploy_script
             
 def load_yaml_config(node_name, yaml_dir='./data', fragment_dir='./stack-builder/fragments', scenario='2_role'):
@@ -31,14 +63,15 @@ def load_yaml_config(node_name, yaml_dir='./data', fragment_dir='./stack-builder
 
         available = available_fragments(fragment_dir)
         for fragment in y['nodes'][node_name]['fragments']:
-            if fragment_dir + '/' + fragment not in available:
+            if fragment_dir + '/' + fragment not in available and not signal_type(fragment):
                 print "Fragment '" + fragment + "' specified in scenario " + scenario + "does not exist "
 
-        return  [ fragment_dir + '/' + f for f in y['nodes'][node_name]['fragments']]
+        return [f for f in y['nodes'][node_name]['fragments']]
+        
 
 def compose(hostname, yaml_dir, fragment_dir, scenario, replacements):
     fragments = load_yaml_config(hostname, yaml_dir, fragment_dir, scenario)
-    script = build_deploy(fragments)
+    script = build_deploy(fragment_dir, fragments, replacements)
     return PercentTemplate(script).safe_substitute(replacements)
 
 def show(n, q, k, args):

stack-builder/fragments/SIGNAL_TEMPLATE

@@ -0,0 +1 @@
+echo %{signal} >> /var/www/status

stack-builder/fragments/WAIT_TEMPLATE

@@ -0,0 +1,6 @@
+for node in %{nodes}; do
+  until [ $(curl http://$node/status | grep %{signal}) ]; do
+    echo "waited for $node" >> /root/waiting
+    sleep 1
+  done
+done

stack-builder/hiera_config.py

@@ -19,32 +19,6 @@ metadata_path = '/root/config.yaml'
 #metadata_path = './sample.json'
 #hiera_dir = './openstack-installer/data/'
 
-def config_builder():
-    # load metadata from config-drive
-    with open(metadata_path, 'r') as metadata:
-       meta = yaml.load(metadata.read())
-       print meta
-
-    # Set values specified in config_drive
-    for path, dirs, files in os.walk(hiera_dir):
-        if '.git' in dirs:
-            dirs.remove('.git')
-        if 'data_mappings' in dirs:
-            dirs.remove('data_mappings')
-        for yaml_file in files:
-            if yaml_file[-5:] == '.yaml':
-                with open(path + '/' + yaml_file, 'r') as hiera_file:
-                    y = yaml.load(hiera_file.read())
-                    for key, value in meta.items():
-                        if (y != None and key in y):
-                            print "Setting : " + key + " with " + str(value)
-                            y[key] = value
-
-                with open(path + '/' + yaml_file, 'w') as hiera_file:
-                    hiera_file.write(yaml.dump(y, default_flow_style=False))
-            
-#config_builder()
-
 # Child processes cannot set environment variables, so
 # create a bash file to set some exports for facter
 def facter_config():
@@ -55,4 +29,12 @@ def facter_config():
             for key,value in meta.items():
                 facts.write('FACTER_' + str(key) + '=' + str(value) + '\n')
 
+#TODO
+def hostname_config():
+    with open(metadata_path, 'r') as metadata:
+        meta = yaml.load(metadata.read())
+        with open('/root/openstack-installer/manifests/setup.pp', 'a') as facts:
+            for key,value in meta.items():
+                pass
+
 facter_config()