diff --git a/devstack-vm-delete.py b/devstack-vm-delete.py
index ea694fec..443ab93f 100755
--- a/devstack-vm-delete.py
+++ b/devstack-vm-delete.py
@@ -28,11 +28,13 @@ import utils
NODE_NAME = sys.argv[1]
+
def main():
db = vmdatabase.VMDatabase()
machine = db.getMachineByJenkinsName(NODE_NAME)
machine.state = vmdatabase.DELETE
+
if __name__ == '__main__':
main()
diff --git a/devstack-vm-fetch.py b/devstack-vm-fetch.py
index a2bbe655..377a3119 100755
--- a/devstack-vm-fetch.py
+++ b/devstack-vm-fetch.py
@@ -25,6 +25,7 @@ import vmdatabase
IMAGE_NAME = sys.argv[1]
+
def main():
db = vmdatabase.VMDatabase()
node = db.getMachineForUse(IMAGE_NAME)
@@ -36,7 +37,10 @@ def main():
gerrit_change = os.environ.get('GERRIT_CHANGE_NUMBER', None)
gerrit_patchset = os.environ.get('GERRIT_PATCHSET_NUMBER', None)
if job_name and build_number and gerrit_change and gerrit_patchset:
- result = node.newResult(job_name, build_number, gerrit_change, gerrit_patchset)
+ result = node.newResult(job_name,
+ build_number,
+ gerrit_change,
+ gerrit_patchset)
else:
result = None
@@ -46,5 +50,6 @@ def main():
if result:
print "RESULT_ID=%s" % result.id
+
if __name__ == "__main__":
main()
diff --git a/devstack-vm-inprogress.py b/devstack-vm-inprogress.py
index 5875ce28..cde7b468 100755
--- a/devstack-vm-inprogress.py
+++ b/devstack-vm-inprogress.py
@@ -32,15 +32,17 @@ import novaclient
import re
NODE_NAME = sys.argv[1]
-DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG',
- os.path.expanduser('~/devstack-gate-secure.conf'))
+DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG',
+ os.path.expanduser(
+ '~/devstack-gate-secure.conf'))
LABEL_RE = re.compile(r'')
+
def main():
db = vmdatabase.VMDatabase()
- config=ConfigParser.ConfigParser()
+ config = ConfigParser.ConfigParser()
config.read(DEVSTACK_GATE_SECURE_CONFIG)
jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'),
@@ -56,7 +58,7 @@ def main():
config = jenkins.get_node_config(machine.jenkins_name)
config = LABEL_RE.sub('', config)
jenkins.reconfig_node(machine.jenkins_name, config)
-
+
if __name__ == '__main__':
main()
diff --git a/devstack-vm-launch.py b/devstack-vm-launch.py
index 148cc1a1..6cec89d8 100755
--- a/devstack-vm-launch.py
+++ b/devstack-vm-launch.py
@@ -33,8 +33,9 @@ import utils
PROVIDER_NAME = sys.argv[1]
DEVSTACK_GATE_PREFIX = os.environ.get('DEVSTACK_GATE_PREFIX', '')
-DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG',
- os.path.expanduser('~/devstack-gate-secure.conf'))
+DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG',
+ os.path.expanduser(
+ '~/devstack-gate-secure.conf'))
SKIP_DEVSTACK_GATE_JENKINS = os.environ.get('SKIP_DEVSTACK_GATE_JENKINS', None)
ABANDON_TIMEOUT = 900 # assume a machine will never boot if it hasn't
@@ -81,22 +82,32 @@ def launch_node(client, snap_image, image, flavor, last_name):
print
return server, machine
+
def create_jenkins_node(jenkins, machine):
- name = '%sdevstack-%s-%s-%s' % (DEVSTACK_GATE_PREFIX, machine.base_image.name,
- machine.base_image.provider.name, machine.id)
+ name = '%sdevstack-%s-%s-%s' % (DEVSTACK_GATE_PREFIX,
+ machine.base_image.name,
+ machine.base_image.provider.name,
+ machine.id)
machine.jenkins_name = name
if jenkins:
- jenkins.create_node(name, numExecutors=1,
- nodeDescription='Dynamic single use %s slave for devstack' % machine.base_image.name,
+ node_desc = 'Dynamic single use %s slave for devstack' % \
+ machine.base_image.name
+ labels = '%sdevstack-%s' % (DEVSTACK_GATE_PREFIX,
+ machine.base_image.name)
+ priv_key = '/var/lib/jenkins/.ssh/id_rsa',
+ jenkins.create_node(name, numExecutors=1,
+ nodeDescription=node_desc,
remoteFS='/home/jenkins',
- labels='%sdevstack-%s' % (DEVSTACK_GATE_PREFIX, machine.base_image.name),
+ labels=labels,
exclusive=True,
launcher='hudson.plugins.sshslaves.SSHLauncher',
- launcher_params = {'port': 22, 'username': 'jenkins',
- 'privatekey': '/var/lib/jenkins/.ssh/id_rsa',
- 'host': machine.ip})
-
+ launcher_params={'port': 22,
+ 'username': 'jenkins',
+ 'privatekey': priv_key,
+ 'host': machine.ip})
+
+
def check_machine(jenkins, client, machine, error_counts):
try:
server = client.servers.get(machine.external_id)
@@ -104,7 +115,7 @@ def check_machine(jenkins, client, machine, error_counts):
print "Unable to get server detail, will retry"
traceback.print_exc()
return
-
+
if server.status == 'ACTIVE':
ip = utils.get_public_ip(server)
if not ip and 'os-floating-ips' in utils.get_extensions(client):
@@ -139,7 +150,7 @@ def main():
db = vmdatabase.VMDatabase()
if not SKIP_DEVSTACK_GATE_JENKINS:
- config=ConfigParser.ConfigParser()
+ config = ConfigParser.ConfigParser()
config.read(DEVSTACK_GATE_SECURE_CONFIG)
jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'),
@@ -173,8 +184,11 @@ def main():
num_to_launch = calculate_deficit(provider, base_image)
for i in range(num_to_launch):
try:
- server, machine = launch_node(client, snap_image,
- remote_snap_image, flavor, last_name)
+ server, machine = launch_node(client,
+ snap_image,
+ remote_snap_image,
+ flavor,
+ last_name)
last_name = machine.name
except:
traceback.print_exc()
diff --git a/devstack-vm-reap.py b/devstack-vm-reap.py
index 3737d84d..5500a51b 100755
--- a/devstack-vm-reap.py
+++ b/devstack-vm-reap.py
@@ -32,8 +32,9 @@ import novaclient
PROVIDER_NAME = sys.argv[1]
MACHINE_LIFETIME = 24 * 60 * 60 # Amount of time after being used
-DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG',
- os.path.expanduser('~/devstack-gate-secure.conf'))
+DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG',
+ os.path.expanduser(
+ '~/devstack-gate-secure.conf'))
SKIP_DEVSTACK_GATE_JENKINS = os.environ.get('SKIP_DEVSTACK_GATE_JENKINS', None)
if '--all-servers' in sys.argv:
@@ -93,7 +94,7 @@ def main():
db = vmdatabase.VMDatabase()
if not SKIP_DEVSTACK_GATE_JENKINS:
- config=ConfigParser.ConfigParser()
+ config = ConfigParser.ConfigParser()
config.read(DEVSTACK_GATE_SECURE_CONFIG)
jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'),
@@ -119,8 +120,9 @@ def main():
for machine in provider.machines:
# Normally, reap machines that have sat in their current state
# for 24 hours, unless that state is READY.
- if (REAP_ALL_SERVERS or (machine.state != vmdatabase.READY and
- now - machine.state_time > MACHINE_LIFETIME) or
+ if (REAP_ALL_SERVERS or
+ (machine.state != vmdatabase.READY and
+ now - machine.state_time > MACHINE_LIFETIME) or
machine.state == vmdatabase.DELETE):
print 'Deleting machine', machine.name
try:
@@ -135,8 +137,9 @@ def main():
for snap_image in base_image.snapshot_images:
# Normally, reap images that have sat in their current state
# for 24 hours, unless the image is the current snapshot
- if REAP_ALL_IMAGES or (snap_image != base_image.current_snapshot and
- now - snap_image.state_time > MACHINE_LIFETIME):
+ if REAP_ALL_IMAGES or \
+ (snap_image != base_image.current_snapshot and
+ now - snap_image.state_time > MACHINE_LIFETIME):
print 'Deleting image', snap_image.name
try:
delete_image(client, snap_image)
diff --git a/devstack-vm-result.py b/devstack-vm-result.py
index ba8f5536..f09d4d98 100755
--- a/devstack-vm-result.py
+++ b/devstack-vm-result.py
@@ -30,6 +30,7 @@ RESULTS = dict(success=vmdatabase.RESULT_SUCCESS,
timeout=vmdatabase.RESULT_TIMEOUT,
)
+
def main():
db = vmdatabase.VMDatabase()
result = db.getResult(RESULT_ID)
diff --git a/devstack-vm-update-image.py b/devstack-vm-update-image.py
index b96f6934..73607c35 100755
--- a/devstack-vm-update-image.py
+++ b/devstack-vm-update-image.py
@@ -48,6 +48,7 @@ PROJECTS = ['openstack/nova',
'openstack-dev/devstack',
'openstack-ci/devstack-gate']
+
def run_local(cmd, status=False, cwd='.', env={}):
print "Running:", cmd
newenv = os.environ
@@ -73,7 +74,8 @@ def tokenize(fn, tokens, distribution, comment=None):
for line in open(fn):
if 'dist:' in line and ('dist:%s' % distribution not in line):
continue
- if 'qpid' in line: continue #XXX
+ if 'qpid' in line:
+ continue # XXX
if comment and comment in line:
line = line[:line.rfind(comment)]
line = line.strip()
@@ -135,49 +137,67 @@ def bootstrap_server(provider, server, admin_pass, key):
ssh_kwargs['pkey'] = key
else:
ssh_kwargs['password'] = admin_pass
-
+
for username in ['root', 'ubuntu']:
client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600)
- if client: break
+ if client:
+ break
if not client:
raise Exception("Unable to log in via SSH")
# hpcloud can't reliably set the hostname
+ gerrit_url = 'https://review.openstack.org/p/openstack/' \
+ 'openstack-ci-puppet.git'
client.ssh("set hostname", "sudo hostname %s" % server.name)
client.ssh("update apt cache", "sudo apt-get update")
client.ssh("upgrading system packages",
- 'sudo DEBIAN_FRONTEND=noninteractive apt-get --option "Dpkg::Options::=--force-confold" --assume-yes upgrade')
+ 'sudo DEBIAN_FRONTEND=noninteractive apt-get '
+ '--option "Dpkg::Options::=--force-confold"'
+ ' --assume-yes upgrade')
client.ssh("install git and puppet",
- 'sudo DEBIAN_FRONTEND=noninteractive apt-get --option "Dpkg::Options::=--force-confold" --assume-yes install git puppet')
+ 'sudo DEBIAN_FRONTEND=noninteractive apt-get '
+ '--option "Dpkg::Options::=--force-confold"'
+ ' --assume-yes install git puppet')
client.ssh("clone puppret repo",
- "sudo git clone https://review.openstack.org/p/openstack/openstack-ci-puppet.git /root/openstack-ci-puppet")
+ "sudo git clone %s /root/openstack-ci-puppet" % gerrit_url)
client.ssh("run puppet",
- "sudo puppet apply --modulepath=/root/openstack-ci-puppet/modules /root/openstack-ci-puppet/manifests/site.pp")
+ "sudo puppet apply "
+ "--modulepath=/root/openstack-ci-puppet/modules"
+ "/root/openstack-ci-puppet/manifests/site.pp")
+
def configure_server(server, branches):
client = SSHClient(utils.get_public_ip(server), 'jenkins')
client.ssh('make file cache directory', 'mkdir -p ~/cache/files')
client.ssh('make pip cache directory', 'mkdir -p ~/cache/pip')
- client.ssh('install build-essential', 'sudo DEBIAN_FRONTEND=noninteractive apt-get --option "Dpkg::Options::=--force-confold" --assume-yes install build-essential python-dev linux-headers-virtual linux-headers-`uname -r`')
+ client.ssh('install build-essential',
+ 'sudo DEBIAN_FRONTEND=noninteractive '
+ 'apt-get --option "Dpkg::Options::=--force-confold"'
+ ' --assume-yes install build-essential python-dev '
+ 'linux-headers-virtual linux-headers-`uname -r`')
for branch_data in branches:
if branch_data['debs']:
client.ssh('cache debs for branch %s' % branch_data['name'],
- 'sudo apt-get -y -d install %s' % ' '.join(branch_data['debs']))
+ 'sudo apt-get -y -d install %s' %
+ ' '.join(branch_data['debs']))
if branch_data['pips']:
venv = client.ssh('get temp dir for venv', 'mktemp -d').strip()
- client.ssh('create venv', 'virtualenv --no-site-packages %s' % venv)
+ client.ssh('create venv',
+ 'virtualenv --no-site-packages %s' % venv)
client.ssh('cache pips for branch %s' % branch_data['name'],
- 'source %s/bin/activate && PIP_DOWNLOAD_CACHE=~/cache/pip pip install %s' %
+ 'source %s/bin/activate && '
+ 'PIP_DOWNLOAD_CACHE=~/cache/pip pip install %s' %
(venv, ' '.join(branch_data['pips'])))
client.ssh('remove venv', 'rm -fr %s' % venv)
for url in branch_data['images']:
fname = url.split('/')[-1]
try:
- client.ssh('check for %s' % fname, 'ls ~/cache/files/%s' % fname)
+ client.ssh('check for %s' % fname,
+ 'ls ~/cache/files/%s' % fname)
except:
client.ssh('download image %s' % fname,
'wget -c %s -O ~/cache/files/%s' % (url, fname))
@@ -187,7 +207,8 @@ def configure_server(server, branches):
for project in PROJECTS:
sp = project.split('/')[0]
client.ssh('clone %s' % project,
- 'cd ~/workspace-cache && git clone https://review.openstack.org/p/%s' % project)
+ 'cd ~/workspace-cache && '
+ 'git clone https://review.openstack.org/p/%s' % project)
script = os.environ.get('DEVSTACK_GATE_CUSTOM_SCRIPT', '')
if script and os.path.isfile(script):
@@ -198,9 +219,10 @@ def configure_server(server, branches):
client.ssh('sync', 'sync && sleep 5')
+
def snapshot_server(client, server, name):
print 'Saving image'
- if hasattr(client.images, 'create'): #v1.0
+ if hasattr(client.images, 'create'): # v1.0
image = client.images.create(server, name)
else:
# TODO: fix novaclient so it returns an image here
@@ -211,7 +233,9 @@ def snapshot_server(client, server, name):
image = utils.wait_for_resource(image)
return image
-def build_image(provider, client, base_image, image, flavor, name, branches, timestamp):
+
+def build_image(provider, client, base_image, image,
+ flavor, name, branches, timestamp):
print "Building image %s" % name
create_kwargs = dict(image=image, flavor=flavor, name=name)
@@ -240,8 +264,8 @@ def build_image(provider, client, base_image, image, flavor, name, branches, tim
# We made the snapshot, try deleting the server, but it's okay
# if we fail. The reap script will find it and try again.
try:
- pass #XXX
- #utils.delete_server(server)
+ pass # XXX
+ # utils.delete_server(server)
except:
print "Exception encountered deleting server:"
traceback.print_exc()
@@ -284,12 +308,13 @@ def main():
remote_base_image = client.images.find(name=base_image.external_id)
timestamp = int(time.time())
remote_snap_image_name = ('%sdevstack-%s-%s.template.openstack.org' %
- (DEVSTACK_GATE_PREFIX, base_image.name, str(timestamp)))
- remote_snap_image = build_image(provider, client, base_image,
- remote_base_image, flavor,
+ (DEVSTACK_GATE_PREFIX,
+ base_image.name, str(timestamp)))
+ remote_snap_image = build_image(provider, client, base_image,
+ remote_base_image, flavor,
remote_snap_image_name,
branches, timestamp)
-
+
if __name__ == '__main__':
main()