b0d8b8be2a
Instead, rely on setting the PYTHONUNBUFFERED env variable in Jenkins. Remove "python" from script invocations, rely on #! instead. Add more output to image update. Exit gate script after deleting server if there are errors. Change-Id: I882cfe0fd3be45c041b37947877c24f043ae4a66
132 lines
5.1 KiB
Python
Executable File
132 lines
5.1 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
# Make sure there are always a certain number of VMs launched and
|
|
# ready for use by devstack.
|
|
|
|
# Copyright (C) 2011 OpenStack LLC.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
#
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from libcloud.base import NodeImage, NodeSize, NodeLocation
|
|
from libcloud.compute.types import NodeState
|
|
from libcloud.types import Provider
|
|
from libcloud.providers import get_driver
|
|
from libcloud.deployment import MultiStepDeployment, ScriptDeployment, SSHKeyDeployment
|
|
import libcloud
|
|
import os, sys
|
|
import getopt
|
|
import time
|
|
|
|
import vmdatabase
|
|
|
|
CLOUD_SERVERS_DRIVER = os.environ.get('CLOUD_SERVERS_DRIVER','rackspace')
|
|
CLOUD_SERVERS_USERNAME = os.environ['CLOUD_SERVERS_USERNAME']
|
|
CLOUD_SERVERS_API_KEY = os.environ['CLOUD_SERVERS_API_KEY']
|
|
IMAGE_NAME = 'devstack-oneiric'
|
|
MIN_RAM = 1024
|
|
MIN_READY_MACHINES = 5
|
|
|
|
db = vmdatabase.VMDatabase()
|
|
|
|
ready_machines = [x for x in db.getMachines()
|
|
if x['state'] == vmdatabase.READY]
|
|
building_machines = [x for x in db.getMachines()
|
|
if x['state'] == vmdatabase.BUILDING]
|
|
|
|
# Count machines that are ready and machines that are building,
|
|
# so that if the provider is very slow, we aren't queueing up tons
|
|
# of machines to be built.
|
|
num_to_launch = MIN_READY_MACHINES - (len(ready_machines) +
|
|
len(building_machines))
|
|
|
|
print "%s ready, %s building, need to launch %s" % (len(ready_machines),
|
|
len(building_machines),
|
|
num_to_launch)
|
|
|
|
if num_to_launch <= 0 and len(building_machines) == 0:
|
|
sys.exit(0)
|
|
|
|
if CLOUD_SERVERS_DRIVER == 'rackspace':
|
|
Driver = get_driver(Provider.RACKSPACE)
|
|
conn = Driver(CLOUD_SERVERS_USERNAME, CLOUD_SERVERS_API_KEY)
|
|
images = conn.list_images()
|
|
|
|
sizes = [sz for sz in conn.list_sizes() if sz.ram >= MIN_RAM]
|
|
sizes.sort(lambda a,b: cmp(a.ram, b.ram))
|
|
size = sizes[0]
|
|
images = [img for img in conn.list_images()
|
|
if img.name.startswith(IMAGE_NAME)]
|
|
images.sort()
|
|
if not len(images):
|
|
raise Exception("No images found")
|
|
image = images[-1]
|
|
else:
|
|
raise Exception ("Driver not supported")
|
|
|
|
if CLOUD_SERVERS_DRIVER == 'rackspace':
|
|
last_name = ''
|
|
error_counts = {}
|
|
for i in range(num_to_launch):
|
|
while True:
|
|
node_name = 'devstack-%s.slave.openstack.org' % int(time.time())
|
|
if node_name != last_name: break
|
|
time.sleep(1)
|
|
node = conn.create_node(name=node_name, image=image, size=size)
|
|
db.addMachine(CLOUD_SERVERS_DRIVER, node.id, IMAGE_NAME,
|
|
node_name, node.public_ip[0], node.uuid)
|
|
print "Started building node %s:" % node.id
|
|
print " name: %s [%s]" % (node_name, node.public_ip[0])
|
|
print " uuid: %s" % (node.uuid)
|
|
print
|
|
|
|
# Wait for nodes
|
|
# TODO: The vmdatabase is (probably) ready, but this needs reworking to
|
|
# actually support multiple providers
|
|
start = time.time()
|
|
timeout = 600
|
|
to_ignore = []
|
|
while (time.time()-start) < timeout:
|
|
building_machines = [x for x in db.getMachines()
|
|
if x['state'] == vmdatabase.BUILDING]
|
|
if not building_machines:
|
|
print "Finished"
|
|
break
|
|
provider_nodes = conn.list_nodes()
|
|
print "Waiting on %s machines" % len(building_machines)
|
|
for my_node in building_machines:
|
|
if my_node['uuid'] in to_ignore: continue
|
|
p_nodes = [x for x in provider_nodes if x.uuid == my_node['uuid']]
|
|
if len(p_nodes) != 1:
|
|
print "Incorrect number of nodes (%s) from provider matching UUID %s" % (len(p_nodes), my_node['uuid'])
|
|
to_ignore.append(my_node)
|
|
else:
|
|
p_node = p_nodes[0]
|
|
if (p_node.public_ips and p_node.state == NodeState.RUNNING):
|
|
print "Node %s is ready" % my_node['id']
|
|
db.setMachineState(my_node['uuid'], vmdatabase.READY)
|
|
if (p_node.public_ips and p_node.state in
|
|
[NodeState.UNKNOWN,
|
|
NodeState.REBOOTING,
|
|
NodeState.TERMINATED]):
|
|
count = error_counts.get(my_node['id'], 0)
|
|
count += 1
|
|
error_counts[my_node['id']] = count
|
|
print "Node %s is in error %s (%s/5)" % (my_node['id'],
|
|
p_node.state,
|
|
count)
|
|
if count >= 5:
|
|
db.setMachineState(my_node['uuid'], vmdatabase.ERROR)
|
|
time.sleep(3)
|