diff --git a/devstack-vm-delete.py b/devstack-vm-delete.py index ea694fec..443ab93f 100755 --- a/devstack-vm-delete.py +++ b/devstack-vm-delete.py @@ -28,11 +28,13 @@ import utils NODE_NAME = sys.argv[1] + def main(): db = vmdatabase.VMDatabase() machine = db.getMachineByJenkinsName(NODE_NAME) machine.state = vmdatabase.DELETE + if __name__ == '__main__': main() diff --git a/devstack-vm-fetch.py b/devstack-vm-fetch.py index a2bbe655..377a3119 100755 --- a/devstack-vm-fetch.py +++ b/devstack-vm-fetch.py @@ -25,6 +25,7 @@ import vmdatabase IMAGE_NAME = sys.argv[1] + def main(): db = vmdatabase.VMDatabase() node = db.getMachineForUse(IMAGE_NAME) @@ -36,7 +37,10 @@ def main(): gerrit_change = os.environ.get('GERRIT_CHANGE_NUMBER', None) gerrit_patchset = os.environ.get('GERRIT_PATCHSET_NUMBER', None) if job_name and build_number and gerrit_change and gerrit_patchset: - result = node.newResult(job_name, build_number, gerrit_change, gerrit_patchset) + result = node.newResult(job_name, + build_number, + gerrit_change, + gerrit_patchset) else: result = None @@ -46,5 +50,6 @@ def main(): if result: print "RESULT_ID=%s" % result.id + if __name__ == "__main__": main() diff --git a/devstack-vm-inprogress.py b/devstack-vm-inprogress.py index 5875ce28..cde7b468 100755 --- a/devstack-vm-inprogress.py +++ b/devstack-vm-inprogress.py @@ -32,15 +32,17 @@ import novaclient import re NODE_NAME = sys.argv[1] -DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG', - os.path.expanduser('~/devstack-gate-secure.conf')) +DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG', + os.path.expanduser( + '~/devstack-gate-secure.conf')) LABEL_RE = re.compile(r'') + def main(): db = vmdatabase.VMDatabase() - config=ConfigParser.ConfigParser() + config = ConfigParser.ConfigParser() config.read(DEVSTACK_GATE_SECURE_CONFIG) jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'), @@ -56,7 +58,7 @@ def main(): config = jenkins.get_node_config(machine.jenkins_name) config = LABEL_RE.sub('', config) jenkins.reconfig_node(machine.jenkins_name, config) - + if __name__ == '__main__': main() diff --git a/devstack-vm-launch.py b/devstack-vm-launch.py index 148cc1a1..6cec89d8 100755 --- a/devstack-vm-launch.py +++ b/devstack-vm-launch.py @@ -33,8 +33,9 @@ import utils PROVIDER_NAME = sys.argv[1] DEVSTACK_GATE_PREFIX = os.environ.get('DEVSTACK_GATE_PREFIX', '') -DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG', - os.path.expanduser('~/devstack-gate-secure.conf')) +DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG', + os.path.expanduser( + '~/devstack-gate-secure.conf')) SKIP_DEVSTACK_GATE_JENKINS = os.environ.get('SKIP_DEVSTACK_GATE_JENKINS', None) ABANDON_TIMEOUT = 900 # assume a machine will never boot if it hasn't @@ -81,22 +82,32 @@ def launch_node(client, snap_image, image, flavor, last_name): print return server, machine + def create_jenkins_node(jenkins, machine): - name = '%sdevstack-%s-%s-%s' % (DEVSTACK_GATE_PREFIX, machine.base_image.name, - machine.base_image.provider.name, machine.id) + name = '%sdevstack-%s-%s-%s' % (DEVSTACK_GATE_PREFIX, + machine.base_image.name, + machine.base_image.provider.name, + machine.id) machine.jenkins_name = name if jenkins: - jenkins.create_node(name, numExecutors=1, - nodeDescription='Dynamic single use %s slave for devstack' % machine.base_image.name, + node_desc = 'Dynamic single use %s slave for devstack' % \ + machine.base_image.name + labels = '%sdevstack-%s' % (DEVSTACK_GATE_PREFIX, + machine.base_image.name) + priv_key = '/var/lib/jenkins/.ssh/id_rsa', + jenkins.create_node(name, numExecutors=1, + nodeDescription=node_desc, remoteFS='/home/jenkins', - labels='%sdevstack-%s' % (DEVSTACK_GATE_PREFIX, machine.base_image.name), + labels=labels, exclusive=True, launcher='hudson.plugins.sshslaves.SSHLauncher', - launcher_params = {'port': 22, 'username': 'jenkins', - 'privatekey': '/var/lib/jenkins/.ssh/id_rsa', - 'host': machine.ip}) - + launcher_params={'port': 22, + 'username': 'jenkins', + 'privatekey': priv_key, + 'host': machine.ip}) + + def check_machine(jenkins, client, machine, error_counts): try: server = client.servers.get(machine.external_id) @@ -104,7 +115,7 @@ def check_machine(jenkins, client, machine, error_counts): print "Unable to get server detail, will retry" traceback.print_exc() return - + if server.status == 'ACTIVE': ip = utils.get_public_ip(server) if not ip and 'os-floating-ips' in utils.get_extensions(client): @@ -139,7 +150,7 @@ def main(): db = vmdatabase.VMDatabase() if not SKIP_DEVSTACK_GATE_JENKINS: - config=ConfigParser.ConfigParser() + config = ConfigParser.ConfigParser() config.read(DEVSTACK_GATE_SECURE_CONFIG) jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'), @@ -173,8 +184,11 @@ def main(): num_to_launch = calculate_deficit(provider, base_image) for i in range(num_to_launch): try: - server, machine = launch_node(client, snap_image, - remote_snap_image, flavor, last_name) + server, machine = launch_node(client, + snap_image, + remote_snap_image, + flavor, + last_name) last_name = machine.name except: traceback.print_exc() diff --git a/devstack-vm-reap.py b/devstack-vm-reap.py index 3737d84d..5500a51b 100755 --- a/devstack-vm-reap.py +++ b/devstack-vm-reap.py @@ -32,8 +32,9 @@ import novaclient PROVIDER_NAME = sys.argv[1] MACHINE_LIFETIME = 24 * 60 * 60 # Amount of time after being used -DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG', - os.path.expanduser('~/devstack-gate-secure.conf')) +DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG', + os.path.expanduser( + '~/devstack-gate-secure.conf')) SKIP_DEVSTACK_GATE_JENKINS = os.environ.get('SKIP_DEVSTACK_GATE_JENKINS', None) if '--all-servers' in sys.argv: @@ -93,7 +94,7 @@ def main(): db = vmdatabase.VMDatabase() if not SKIP_DEVSTACK_GATE_JENKINS: - config=ConfigParser.ConfigParser() + config = ConfigParser.ConfigParser() config.read(DEVSTACK_GATE_SECURE_CONFIG) jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'), @@ -119,8 +120,9 @@ def main(): for machine in provider.machines: # Normally, reap machines that have sat in their current state # for 24 hours, unless that state is READY. - if (REAP_ALL_SERVERS or (machine.state != vmdatabase.READY and - now - machine.state_time > MACHINE_LIFETIME) or + if (REAP_ALL_SERVERS or + (machine.state != vmdatabase.READY and + now - machine.state_time > MACHINE_LIFETIME) or machine.state == vmdatabase.DELETE): print 'Deleting machine', machine.name try: @@ -135,8 +137,9 @@ def main(): for snap_image in base_image.snapshot_images: # Normally, reap images that have sat in their current state # for 24 hours, unless the image is the current snapshot - if REAP_ALL_IMAGES or (snap_image != base_image.current_snapshot and - now - snap_image.state_time > MACHINE_LIFETIME): + if REAP_ALL_IMAGES or \ + (snap_image != base_image.current_snapshot and + now - snap_image.state_time > MACHINE_LIFETIME): print 'Deleting image', snap_image.name try: delete_image(client, snap_image) diff --git a/devstack-vm-result.py b/devstack-vm-result.py index ba8f5536..f09d4d98 100755 --- a/devstack-vm-result.py +++ b/devstack-vm-result.py @@ -30,6 +30,7 @@ RESULTS = dict(success=vmdatabase.RESULT_SUCCESS, timeout=vmdatabase.RESULT_TIMEOUT, ) + def main(): db = vmdatabase.VMDatabase() result = db.getResult(RESULT_ID) diff --git a/devstack-vm-update-image.py b/devstack-vm-update-image.py index b96f6934..73607c35 100755 --- a/devstack-vm-update-image.py +++ b/devstack-vm-update-image.py @@ -48,6 +48,7 @@ PROJECTS = ['openstack/nova', 'openstack-dev/devstack', 'openstack-ci/devstack-gate'] + def run_local(cmd, status=False, cwd='.', env={}): print "Running:", cmd newenv = os.environ @@ -73,7 +74,8 @@ def tokenize(fn, tokens, distribution, comment=None): for line in open(fn): if 'dist:' in line and ('dist:%s' % distribution not in line): continue - if 'qpid' in line: continue #XXX + if 'qpid' in line: + continue # XXX if comment and comment in line: line = line[:line.rfind(comment)] line = line.strip() @@ -135,49 +137,67 @@ def bootstrap_server(provider, server, admin_pass, key): ssh_kwargs['pkey'] = key else: ssh_kwargs['password'] = admin_pass - + for username in ['root', 'ubuntu']: client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) - if client: break + if client: + break if not client: raise Exception("Unable to log in via SSH") # hpcloud can't reliably set the hostname + gerrit_url = 'https://review.openstack.org/p/openstack/' \ + 'openstack-ci-puppet.git' client.ssh("set hostname", "sudo hostname %s" % server.name) client.ssh("update apt cache", "sudo apt-get update") client.ssh("upgrading system packages", - 'sudo DEBIAN_FRONTEND=noninteractive apt-get --option "Dpkg::Options::=--force-confold" --assume-yes upgrade') + 'sudo DEBIAN_FRONTEND=noninteractive apt-get ' + '--option "Dpkg::Options::=--force-confold"' + ' --assume-yes upgrade') client.ssh("install git and puppet", - 'sudo DEBIAN_FRONTEND=noninteractive apt-get --option "Dpkg::Options::=--force-confold" --assume-yes install git puppet') + 'sudo DEBIAN_FRONTEND=noninteractive apt-get ' + '--option "Dpkg::Options::=--force-confold"' + ' --assume-yes install git puppet') client.ssh("clone puppret repo", - "sudo git clone https://review.openstack.org/p/openstack/openstack-ci-puppet.git /root/openstack-ci-puppet") + "sudo git clone %s /root/openstack-ci-puppet" % gerrit_url) client.ssh("run puppet", - "sudo puppet apply --modulepath=/root/openstack-ci-puppet/modules /root/openstack-ci-puppet/manifests/site.pp") + "sudo puppet apply " + "--modulepath=/root/openstack-ci-puppet/modules" + "/root/openstack-ci-puppet/manifests/site.pp") + def configure_server(server, branches): client = SSHClient(utils.get_public_ip(server), 'jenkins') client.ssh('make file cache directory', 'mkdir -p ~/cache/files') client.ssh('make pip cache directory', 'mkdir -p ~/cache/pip') - client.ssh('install build-essential', 'sudo DEBIAN_FRONTEND=noninteractive apt-get --option "Dpkg::Options::=--force-confold" --assume-yes install build-essential python-dev linux-headers-virtual linux-headers-`uname -r`') + client.ssh('install build-essential', + 'sudo DEBIAN_FRONTEND=noninteractive ' + 'apt-get --option "Dpkg::Options::=--force-confold"' + ' --assume-yes install build-essential python-dev ' + 'linux-headers-virtual linux-headers-`uname -r`') for branch_data in branches: if branch_data['debs']: client.ssh('cache debs for branch %s' % branch_data['name'], - 'sudo apt-get -y -d install %s' % ' '.join(branch_data['debs'])) + 'sudo apt-get -y -d install %s' % + ' '.join(branch_data['debs'])) if branch_data['pips']: venv = client.ssh('get temp dir for venv', 'mktemp -d').strip() - client.ssh('create venv', 'virtualenv --no-site-packages %s' % venv) + client.ssh('create venv', + 'virtualenv --no-site-packages %s' % venv) client.ssh('cache pips for branch %s' % branch_data['name'], - 'source %s/bin/activate && PIP_DOWNLOAD_CACHE=~/cache/pip pip install %s' % + 'source %s/bin/activate && ' + 'PIP_DOWNLOAD_CACHE=~/cache/pip pip install %s' % (venv, ' '.join(branch_data['pips']))) client.ssh('remove venv', 'rm -fr %s' % venv) for url in branch_data['images']: fname = url.split('/')[-1] try: - client.ssh('check for %s' % fname, 'ls ~/cache/files/%s' % fname) + client.ssh('check for %s' % fname, + 'ls ~/cache/files/%s' % fname) except: client.ssh('download image %s' % fname, 'wget -c %s -O ~/cache/files/%s' % (url, fname)) @@ -187,7 +207,8 @@ def configure_server(server, branches): for project in PROJECTS: sp = project.split('/')[0] client.ssh('clone %s' % project, - 'cd ~/workspace-cache && git clone https://review.openstack.org/p/%s' % project) + 'cd ~/workspace-cache && ' + 'git clone https://review.openstack.org/p/%s' % project) script = os.environ.get('DEVSTACK_GATE_CUSTOM_SCRIPT', '') if script and os.path.isfile(script): @@ -198,9 +219,10 @@ def configure_server(server, branches): client.ssh('sync', 'sync && sleep 5') + def snapshot_server(client, server, name): print 'Saving image' - if hasattr(client.images, 'create'): #v1.0 + if hasattr(client.images, 'create'): # v1.0 image = client.images.create(server, name) else: # TODO: fix novaclient so it returns an image here @@ -211,7 +233,9 @@ def snapshot_server(client, server, name): image = utils.wait_for_resource(image) return image -def build_image(provider, client, base_image, image, flavor, name, branches, timestamp): + +def build_image(provider, client, base_image, image, + flavor, name, branches, timestamp): print "Building image %s" % name create_kwargs = dict(image=image, flavor=flavor, name=name) @@ -240,8 +264,8 @@ def build_image(provider, client, base_image, image, flavor, name, branches, tim # We made the snapshot, try deleting the server, but it's okay # if we fail. The reap script will find it and try again. try: - pass #XXX - #utils.delete_server(server) + pass # XXX + # utils.delete_server(server) except: print "Exception encountered deleting server:" traceback.print_exc() @@ -284,12 +308,13 @@ def main(): remote_base_image = client.images.find(name=base_image.external_id) timestamp = int(time.time()) remote_snap_image_name = ('%sdevstack-%s-%s.template.openstack.org' % - (DEVSTACK_GATE_PREFIX, base_image.name, str(timestamp))) - remote_snap_image = build_image(provider, client, base_image, - remote_base_image, flavor, + (DEVSTACK_GATE_PREFIX, + base_image.name, str(timestamp))) + remote_snap_image = build_image(provider, client, base_image, + remote_base_image, flavor, remote_snap_image_name, branches, timestamp) - + if __name__ == '__main__': main()