openstack/devstack-gate
Code Issues Proposed changes

Pep8 fixes, found with tox.

Browse Source 
* devstack-vm-delete.py
* devstack-vm-fetch.py
* devstack-vm-inprogress.py
* devstack-vm-launch.py
* devstack-vm-reap.py
* devstack-vm-result.py
* devstack-vm-update-image.py
  No functional change, just pep8 fixes.

Change-Id: I8a058519dfabceaed2aac72565a8c1a2e91f8806
This commit is contained in:
Bhuvan Arumugam
2012-07-08 15:12:41 -07:00
parent 17d5310a62
commit 634286eced
7 changed files with 100 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
devstack-vm-delete.py
View File

@@ -28,11 +28,13 @@ import utils
NODE_NAME = sys.argv[1]
def main():
db = vmdatabase.VMDatabase()
machine = db.getMachineByJenkinsName(NODE_NAME)
machine.state = vmdatabase.DELETE
if __name__ == '__main__':
main()

7
devstack-vm-fetch.py
View File

@@ -25,6 +25,7 @@ import vmdatabase
IMAGE_NAME = sys.argv[1]
def main():
db = vmdatabase.VMDatabase()
node = db.getMachineForUse(IMAGE_NAME)
@@ -36,7 +37,10 @@ def main():
gerrit_change = os.environ.get('GERRIT_CHANGE_NUMBER', None)
gerrit_patchset = os.environ.get('GERRIT_PATCHSET_NUMBER', None)
if job_name and build_number and gerrit_change and gerrit_patchset:
result = node.newResult(job_name, build_number, gerrit_change, gerrit_patchset)
result = node.newResult(job_name,
build_number,
gerrit_change,
gerrit_patchset)
else:
result = None
@@ -46,5 +50,6 @@ def main():
if result:
print "RESULT_ID=%s" % result.id
if __name__ == "__main__":
main()

10
devstack-vm-inprogress.py
View File

@@ -32,15 +32,17 @@ import novaclient
import re
NODE_NAME = sys.argv[1]
DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG',
os.path.expanduser('~/devstack-gate-secure.conf'))
DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG',
os.path.expanduser(
'~/devstack-gate-secure.conf'))
LABEL_RE = re.compile(r'<label>.*</label>')
def main():
db = vmdatabase.VMDatabase()
config=ConfigParser.ConfigParser()
config = ConfigParser.ConfigParser()
config.read(DEVSTACK_GATE_SECURE_CONFIG)
jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'),
@@ -56,7 +58,7 @@ def main():
config = jenkins.get_node_config(machine.jenkins_name)
config = LABEL_RE.sub('<label>devstack-used</label>', config)
jenkins.reconfig_node(machine.jenkins_name, config)
if __name__ == '__main__':
main()

44
devstack-vm-launch.py
View File

@@ -33,8 +33,9 @@ import utils
PROVIDER_NAME = sys.argv[1]
DEVSTACK_GATE_PREFIX = os.environ.get('DEVSTACK_GATE_PREFIX', '')
DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG',
os.path.expanduser('~/devstack-gate-secure.conf'))
DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG',
os.path.expanduser(
'~/devstack-gate-secure.conf'))
SKIP_DEVSTACK_GATE_JENKINS = os.environ.get('SKIP_DEVSTACK_GATE_JENKINS', None)
ABANDON_TIMEOUT = 900 # assume a machine will never boot if it hasn't
@@ -81,22 +82,32 @@ def launch_node(client, snap_image, image, flavor, last_name):
print
return server, machine
def create_jenkins_node(jenkins, machine):
name = '%sdevstack-%s-%s-%s' % (DEVSTACK_GATE_PREFIX, machine.base_image.name,
machine.base_image.provider.name, machine.id)
name = '%sdevstack-%s-%s-%s' % (DEVSTACK_GATE_PREFIX,
machine.base_image.name,
machine.base_image.provider.name,
machine.id)
machine.jenkins_name = name
if jenkins:
jenkins.create_node(name, numExecutors=1,
nodeDescription='Dynamic single use %s slave for devstack' % machine.base_image.name,
node_desc = 'Dynamic single use %s slave for devstack' % \
machine.base_image.name
labels = '%sdevstack-%s' % (DEVSTACK_GATE_PREFIX,
machine.base_image.name)
priv_key = '/var/lib/jenkins/.ssh/id_rsa',
jenkins.create_node(name, numExecutors=1,
nodeDescription=node_desc,
remoteFS='/home/jenkins',
labels='%sdevstack-%s' % (DEVSTACK_GATE_PREFIX, machine.base_image.name),
labels=labels,
exclusive=True,
launcher='hudson.plugins.sshslaves.SSHLauncher',
launcher_params = {'port': 22, 'username': 'jenkins',
'privatekey': '/var/lib/jenkins/.ssh/id_rsa',
'host': machine.ip})
launcher_params={'port': 22,
'username': 'jenkins',
'privatekey': priv_key,
'host': machine.ip})
def check_machine(jenkins, client, machine, error_counts):
try:
server = client.servers.get(machine.external_id)
@@ -104,7 +115,7 @@ def check_machine(jenkins, client, machine, error_counts):
print "Unable to get server detail, will retry"
traceback.print_exc()
return
if server.status == 'ACTIVE':
ip = utils.get_public_ip(server)
if not ip and 'os-floating-ips' in utils.get_extensions(client):
@@ -139,7 +150,7 @@ def main():
db = vmdatabase.VMDatabase()
if not SKIP_DEVSTACK_GATE_JENKINS:
config=ConfigParser.ConfigParser()
config = ConfigParser.ConfigParser()
config.read(DEVSTACK_GATE_SECURE_CONFIG)
jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'),
@@ -173,8 +184,11 @@ def main():
num_to_launch = calculate_deficit(provider, base_image)
for i in range(num_to_launch):
try:
server, machine = launch_node(client, snap_image,
remote_snap_image, flavor, last_name)
server, machine = launch_node(client,
snap_image,
remote_snap_image,
flavor,
last_name)
last_name = machine.name
except:
traceback.print_exc()

17
devstack-vm-reap.py
View File

@@ -32,8 +32,9 @@ import novaclient
PROVIDER_NAME = sys.argv[1]
MACHINE_LIFETIME = 24 * 60 * 60 # Amount of time after being used
DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG',
os.path.expanduser('~/devstack-gate-secure.conf'))
DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG',
os.path.expanduser(
'~/devstack-gate-secure.conf'))
SKIP_DEVSTACK_GATE_JENKINS = os.environ.get('SKIP_DEVSTACK_GATE_JENKINS', None)
if '--all-servers' in sys.argv:
@@ -93,7 +94,7 @@ def main():
db = vmdatabase.VMDatabase()
if not SKIP_DEVSTACK_GATE_JENKINS:
config=ConfigParser.ConfigParser()
config = ConfigParser.ConfigParser()
config.read(DEVSTACK_GATE_SECURE_CONFIG)
jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'),
@@ -119,8 +120,9 @@ def main():
for machine in provider.machines:
# Normally, reap machines that have sat in their current state
# for 24 hours, unless that state is READY.
if (REAP_ALL_SERVERS or (machine.state != vmdatabase.READY and
now - machine.state_time > MACHINE_LIFETIME) or
if (REAP_ALL_SERVERS or
(machine.state != vmdatabase.READY and
now - machine.state_time > MACHINE_LIFETIME) or
machine.state == vmdatabase.DELETE):
print 'Deleting machine', machine.name
try:
@@ -135,8 +137,9 @@ def main():
for snap_image in base_image.snapshot_images:
# Normally, reap images that have sat in their current state
# for 24 hours, unless the image is the current snapshot
if REAP_ALL_IMAGES or (snap_image != base_image.current_snapshot and
now - snap_image.state_time > MACHINE_LIFETIME):
if REAP_ALL_IMAGES or \
(snap_image != base_image.current_snapshot and
now - snap_image.state_time > MACHINE_LIFETIME):
print 'Deleting image', snap_image.name
try:
delete_image(client, snap_image)

1
devstack-vm-result.py
View File

@@ -30,6 +30,7 @@ RESULTS = dict(success=vmdatabase.RESULT_SUCCESS,
timeout=vmdatabase.RESULT_TIMEOUT,
)
def main():
db = vmdatabase.VMDatabase()
result = db.getResult(RESULT_ID)

67
devstack-vm-update-image.py
View File

@@ -48,6 +48,7 @@ PROJECTS = ['openstack/nova',
'openstack-dev/devstack',
'openstack-ci/devstack-gate']
def run_local(cmd, status=False, cwd='.', env={}):
print "Running:", cmd
newenv = os.environ
@@ -73,7 +74,8 @@ def tokenize(fn, tokens, distribution, comment=None):
for line in open(fn):
if 'dist:' in line and ('dist:%s' % distribution not in line):
continue
if 'qpid' in line: continue #XXX
if 'qpid' in line:
continue # XXX
if comment and comment in line:
line = line[:line.rfind(comment)]
line = line.strip()
@@ -135,49 +137,67 @@ def bootstrap_server(provider, server, admin_pass, key):
ssh_kwargs['pkey'] = key
else:
ssh_kwargs['password'] = admin_pass
for username in ['root', 'ubuntu']:
client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600)
if client: break
if client:
break
if not client:
raise Exception("Unable to log in via SSH")
# hpcloud can't reliably set the hostname
gerrit_url = 'https://review.openstack.org/p/openstack/' \
'openstack-ci-puppet.git'
client.ssh("set hostname", "sudo hostname %s" % server.name)
client.ssh("update apt cache", "sudo apt-get update")
client.ssh("upgrading system packages",
'sudo DEBIAN_FRONTEND=noninteractive apt-get --option "Dpkg::Options::=--force-confold" --assume-yes upgrade')
'sudo DEBIAN_FRONTEND=noninteractive apt-get '
'--option "Dpkg::Options::=--force-confold"'
' --assume-yes upgrade')
client.ssh("install git and puppet",
'sudo DEBIAN_FRONTEND=noninteractive apt-get --option "Dpkg::Options::=--force-confold" --assume-yes install git puppet')
'sudo DEBIAN_FRONTEND=noninteractive apt-get '
'--option "Dpkg::Options::=--force-confold"'
' --assume-yes install git puppet')
client.ssh("clone puppret repo",
"sudo git clone https://review.openstack.org/p/openstack/openstack-ci-puppet.git /root/openstack-ci-puppet")
"sudo git clone %s /root/openstack-ci-puppet" % gerrit_url)
client.ssh("run puppet",
"sudo puppet apply --modulepath=/root/openstack-ci-puppet/modules /root/openstack-ci-puppet/manifests/site.pp")
"sudo puppet apply "
"--modulepath=/root/openstack-ci-puppet/modules"
"/root/openstack-ci-puppet/manifests/site.pp")
def configure_server(server, branches):
client = SSHClient(utils.get_public_ip(server), 'jenkins')
client.ssh('make file cache directory', 'mkdir -p ~/cache/files')
client.ssh('make pip cache directory', 'mkdir -p ~/cache/pip')
client.ssh('install build-essential', 'sudo DEBIAN_FRONTEND=noninteractive apt-get --option "Dpkg::Options::=--force-confold" --assume-yes install build-essential python-dev linux-headers-virtual linux-headers-`uname -r`')
client.ssh('install build-essential',
'sudo DEBIAN_FRONTEND=noninteractive '
'apt-get --option "Dpkg::Options::=--force-confold"'
' --assume-yes install build-essential python-dev '
'linux-headers-virtual linux-headers-`uname -r`')
for branch_data in branches:
if branch_data['debs']:
client.ssh('cache debs for branch %s' % branch_data['name'],
'sudo apt-get -y -d install %s' % ' '.join(branch_data['debs']))
'sudo apt-get -y -d install %s' %
' '.join(branch_data['debs']))
if branch_data['pips']:
venv = client.ssh('get temp dir for venv', 'mktemp -d').strip()
client.ssh('create venv', 'virtualenv --no-site-packages %s' % venv)
client.ssh('create venv',
'virtualenv --no-site-packages %s' % venv)
client.ssh('cache pips for branch %s' % branch_data['name'],
'source %s/bin/activate && PIP_DOWNLOAD_CACHE=~/cache/pip pip install %s' %
'source %s/bin/activate && '
'PIP_DOWNLOAD_CACHE=~/cache/pip pip install %s' %
(venv, ' '.join(branch_data['pips'])))
client.ssh('remove venv', 'rm -fr %s' % venv)
for url in branch_data['images']:
fname = url.split('/')[-1]
try:
client.ssh('check for %s' % fname, 'ls ~/cache/files/%s' % fname)
client.ssh('check for %s' % fname,
'ls ~/cache/files/%s' % fname)
except:
client.ssh('download image %s' % fname,
'wget -c %s -O ~/cache/files/%s' % (url, fname))
@@ -187,7 +207,8 @@ def configure_server(server, branches):
for project in PROJECTS:
sp = project.split('/')[0]
client.ssh('clone %s' % project,
'cd ~/workspace-cache && git clone https://review.openstack.org/p/%s' % project)
'cd ~/workspace-cache && '
'git clone https://review.openstack.org/p/%s' % project)
script = os.environ.get('DEVSTACK_GATE_CUSTOM_SCRIPT', '')
if script and os.path.isfile(script):
@@ -198,9 +219,10 @@ def configure_server(server, branches):
client.ssh('sync', 'sync && sleep 5')
def snapshot_server(client, server, name):
print 'Saving image'
if hasattr(client.images, 'create'): #v1.0
if hasattr(client.images, 'create'): # v1.0
image = client.images.create(server, name)
else:
# TODO: fix novaclient so it returns an image here
@@ -211,7 +233,9 @@ def snapshot_server(client, server, name):
image = utils.wait_for_resource(image)
return image
def build_image(provider, client, base_image, image, flavor, name, branches, timestamp):
def build_image(provider, client, base_image, image,
flavor, name, branches, timestamp):
print "Building image %s" % name
create_kwargs = dict(image=image, flavor=flavor, name=name)
@@ -240,8 +264,8 @@ def build_image(provider, client, base_image, image, flavor, name, branches, tim
# We made the snapshot, try deleting the server, but it's okay
# if we fail. The reap script will find it and try again.
try:
pass #XXX
#utils.delete_server(server)
pass # XXX
# utils.delete_server(server)
except:
print "Exception encountered deleting server:"
traceback.print_exc()
@@ -284,12 +308,13 @@ def main():
remote_base_image = client.images.find(name=base_image.external_id)
timestamp = int(time.time())
remote_snap_image_name = ('%sdevstack-%s-%s.template.openstack.org' %
(DEVSTACK_GATE_PREFIX, base_image.name, str(timestamp)))
remote_snap_image = build_image(provider, client, base_image,
remote_base_image, flavor,
(DEVSTACK_GATE_PREFIX,
base_image.name, str(timestamp)))
remote_snap_image = build_image(provider, client, base_image,
remote_base_image, flavor,
remote_snap_image_name,
branches, timestamp)
if __name__ == '__main__':
main()