Add support for moving between clusters.

DNS server changes
Removal of keystone commands that needed admin
Hack to avoid dhcp scheduling issues

README.md

@@ -121,30 +121,6 @@ To create a basic 2 role cluster with a build, compute and control node outside
     
     e824830b269544d39a632d89e0a1902c
 
-To destroy all resources created by the sb tool:
-
-    sb kill
-
-    deleted port b68c7ff8-7598-42fd-ac3c-2e2b7021d2c6
-    deleted port baa3c279-2b3d-4dad-a76a-f699de96d629
-    deleted port c6603b5d-cc0c-47be-a897-e667727294ae
-    deleted subnetci-9cbbfa7d10b54ff0b87e5983a492e05c22
-    deleted subnetci-9cbbfa7d10b54ff0b87e5983a492e05c11
-    deleted networkci-9cbbfa7d10b54ff0b87e5983a492e05c1
-    deleted networkci-9cbbfa7d10b54ff0b87e5983a492e05c2
-
-To destroy a specific test run's resources:
-
-    sb kill -t e824830b269544d39a632d89e0a1902c
-
-    deleted port b68c7ff8-7598-42fd-ac3c-2e2b7021d2c6
-    deleted port baa3c279-2b3d-4dad-a76a-f699de96d629
-    deleted port c6603b5d-cc0c-47be-a897-e667727294ae
-    deleted subnetci-9cbbfa7d10b54ff0b87e5983a492e05c22
-    deleted subnetci-9cbbfa7d10b54ff0b87e5983a492e05c11
-    deleted networkci-9cbbfa7d10b54ff0b87e5983a492e05c1
-    deleted networkci-9cbbfa7d10b54ff0b87e5983a492e05c2
-
 More information about this tool can be found under the stack-builder directory.
 
 ## basic install against already provisioned nodes:

stack-builder/bin/sb

@@ -31,6 +31,7 @@ def main():
     parser_make.add_argument('-f', '--fragment_path', default='./stack-builder/fragments', help='Path to config fragments')
     parser_make.add_argument('-d', '--debug', action='store_true', help='print debug output')
     parser_make.add_argument('-n', '--public_network', default='public', help='The name of the public network on the cluster')
+    parser_make.add_argument('-a', '--nameserver', default='171.70.168.183', help='The DNS nameserver to use for all nodes being booted.')
     parser_make.set_defaults(func=make)
 
     parser_rebuild = subparsers.add_parser('rebuild', help='Rebuild a test using existing resources')

stack-builder/build.py

@@ -51,10 +51,6 @@ def build_nic_port_list(ports):
 def make_network(q, ci_network_name, index=0):
     networks = q.list_networks()
 
-    # the shared CI network 
-    #if ci_network_name != 'ci':
-    #    ci_network_name = ci_network_name + str(index)
-
     if ci_network_name not in [network['name'] for network in networks['networks']]:
         dprint("q.create_network({'network': {'name':" + ci_network_name + ", 'admin_state_up': True}})['network']")
         test_net = q.create_network({'network': {'name': ci_network_name, 'admin_state_up': True}})['network']
@@ -65,13 +61,9 @@ def make_network(q, ci_network_name, index=0):
               test_net = net
     return test_net
 
-def make_subnet(q, ci_network_name, test_net, index=1, dhcp=True, gateway=False):
+def make_subnet(q, ci_network_name, test_net, index=1, dhcp=True, gateway=False, dns_nameserver="171.70.168.183"):
     subnets = q.list_subnets()
 
-    # the shared CI network 
-    #if ci_network_name != 'ci':
-    #    ci_network_name = ci_network_name + str(index)
-
     if ci_network_name not in [subnet['name'] for subnet in subnets['subnets']]:
         dprint("CI subnet " + str(index) + " doesn't exist. Creating ...")
         try:
@@ -99,17 +91,17 @@ def make_subnet(q, ci_network_name, test_net, index=1, dhcp=True, gateway=False)
                            "            'network_id': " + test_net['id'] + ",\n" +
                            "            'ip_version': 4,\n" +
                            "            'cidr': '10." + str(index) + ".0.0/16',\n" +
-                           "            'enable_dhcp':" + dhcp + ",\n" +
+                           "            'enable_dhcp':" + str(dhcp) + ",\n" +
-                           "            'gateway_ip': '10.'" + str(index) + ".0.1\n" +
+                           "            'gateway_ip': '10." + str(index) + ".0.1'\n" +
-                           "            'dns_nameservers': ['171.70.168.183']\n")
+                           "            'dns_nameservers: ['" + dns_nameserver + "']\n")
 
                     test_subnet = q.create_subnet({'subnet': { 'name': ci_network_name, 
                                          'network_id': test_net['id'],
                                          'ip_version': 4,
                                          'cidr': '10.' + str(index) + '.0.0/16',
                                          'enable_dhcp': dhcp,
-                                         'gateway_ip' : '10.' + str(index) + '.0.1',
+                                         #'gateway_ip' : '10.' + str(index) + '.0.1',
-                                         'dns_nameservers': ['171.70.168.183']
+                                         'dns_nameservers': [unicode(dns_nameserver)]
 	                                 }})['subnet']
 
         except quantumclient.common.exceptions.QuantumClientException:
@@ -152,7 +144,7 @@ def get_external_network(q):
 # belong on external network
 def get_public_network(q, public_network):
     for network in q.list_networks()['networks']:
-        if network['name'] == public_network:
+        if network['name'] == unicode(public_network):
             return network
 
 def get_external_router(q):
@@ -160,22 +152,19 @@ def get_external_router(q):
         if router['name'] == 'ci':
             return router
 
-def get_tenant_id(k):
+def get_tenant_id(network):
-    tenant_name = os.environ['OS_TENANT_NAME']
+    return str(network['tenant_id'])
-    for tenant in k.tenants.list():
-        if tenant.name == tenant_name:
-            return str(tenant.id)
 
 # This can easily be problematic with multiple tenants,
 # So make sure we get the one for the current tenant
-def get_ci_network(q, k):
+def get_ci_network(q, n):
     for network in q.list_networks()['networks']:
-        if network['name'] == 'ci' and network['tenant_id'] == get_tenant_id(k):
+        if network['name'] == 'ci' and network['tenant_id'] == get_tenant_id(n):
             return network
 
-def get_ci_subnet(q, k):
+def get_ci_subnet(q, n):
     for subnet in q.list_subnets()['subnets']:
-        if subnet['name'] == 'ci' and subnet['tenant_id'] == get_tenant_id(k):
+        if subnet['name'] == 'ci' and subnet['tenant_id'] == get_tenant_id(n):
             return subnet 
 
 def set_external_routing(q, subnet, public_network):
@@ -226,6 +215,7 @@ def make(n, q, k, args):
     data_path       = args.data_path
     fragment_path   = args.fragment_path
     public_network  = args.public_network
+    nameserver      = args.nameserver
 
     if args.debug:
         debug.debug = True
@@ -241,8 +231,8 @@ def make(n, q, k, args):
     # There can be only one of these per tenant
     # because overlapping subnets + router doesn't work
     networks['ci'] = make_network(q, 'ci')
-    subnets['ci']  = make_subnet(q, 'ci', networks['ci'], ci_subnet_index, gateway=True)
+    subnets['ci']  = make_subnet(q, 'ci', networks['ci'], ci_subnet_index, gateway=True, dns_nameserver=nameserver)
-    set_external_routing(q, get_ci_subnet(q, k), public_network)
+    set_external_routing(q, get_ci_subnet(q, networks['ci']), public_network)
     ci_subnet_index = ci_subnet_index + 1
 
     with open(data_path + '/nodes/' + scenario + '.yaml') as scenario_yaml_file:
@@ -254,6 +244,7 @@ def make(n, q, k, args):
             if network != 'ci': # build network with NAT services
                 networks[network] = False
 
+
     # Create internal networks
     for network, gate in networks.items():
         if network != 'ci':
@@ -262,6 +253,23 @@ def make(n, q, k, args):
                                         networks[network], index=ci_subnet_index, gateway=gate)
             ci_subnet_index = ci_subnet_index + 1
 
+    # There seems to be a bug in quantum where networks are not scheduled a dhcp agent unless a VM
+    # boots on that network without a pre-made port. So we boot an instance that will do this
+    # on all our networks
+    dummynets = [network for network in networks.values()]
+    dummy = boot_puppetised_instance(n,
+                        'dummy',
+                        image,
+                        build_nic_net_list(dummynets),
+                        deploy=cloud_init,
+                        meta={'ci_test_id' : test_id},
+                        os_flavor=u'm1.small'
+                        )
+    while dummy.status != u'ACTIVE':
+        dummy = n.servers.get(dummy)
+        dprint('dummy status: ' + str(dummy.status))
+    dummy.delete()
+
     # Allocate ports
     for node, props in scenario_yaml['nodes'].items():
         for network in props['networks']:

stack-builder/fragments/pw-mirror-network

@@ -1,7 +1,9 @@
 usermod --password p1fhTXKKhbc0M root
 
 # Rstarmer's precise mirror
-sed -i 's/nova.clouds.archive.ubuntu.com/172.29.74.100/g' /etc/apt/sources.list
+# Different ubuntu images have different defaults
+sed -i 's/nova.clouds.archive.ubuntu.com/%{apt_mirror_ip}/g' /etc/apt/sources.list
+sed -i 's/archive.ubuntu.com/%{apt_mirror_ip}/g' /etc/apt/sources.list
 
 for i in 1 2 3
 do