diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 5e5b635f..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,9 +0,0 @@ -include AUTHORS -include ChangeLog -include README.md -include setup.cfg -include setup.py -include tox.ini -graft tests -graft tools -global-exclude *.pyc *.sdx *.log *.db *.swp diff --git a/README.rst b/README.rst index 26e681e7..6bbbf309 100644 --- a/README.rst +++ b/README.rst @@ -14,24 +14,9 @@ passes all of the configured tests. Most projects require unit tests in python2.6 and python2.7, and pep8. Those tests are all run only on the project in question. The devstack gate test, however, is an integration test and ensures that a proposed change still enables -several of the projects to work together. Currently, any proposed -change to the following projects must pass the devstack gate test:: +several of the projects to work together. - nova - glance - keystone - heat - horizon - neutron - ceilometer - python-novaclient - python-heatclient - python-keystoneclient - python-neutronclient - devstack - devstack-gate - -Obviously we test nova, glance, keystone, horizon, neutron and their clients +Obviously we test integrated OpenStack components and their clients because they all work closely together to form an OpenStack system. Changes to devstack itself are also required to pass this test so that we can be assured that devstack is always able to produce a @@ -42,107 +27,18 @@ How It Works ============ The devstack test starts with an essentially bare virtual machine, -installs devstack on it, and runs some simple tests of the resulting -OpenStack installation. In order to ensure that each test run is -independent, the virtual machine is discarded at the end of the run, -and a new machine is used for the next run. In order to keep the -actual test run as short and reliable as possible, the virtual -machines are prepared ahead of time and kept in a pool ready for -immediate use. The process of preparing the machines ahead of time -reduces network traffic and external dependencies during the run. +installs devstack on it, and runs tests of the resulting OpenStack +installation. In order to ensure that each test run is independent, +the virtual machine is discarded at the end of the run, and a new +machine is used for the next run. In order to keep the actual test run +as short and reliable as possible, the virtual machines are prepared +ahead of time and kept in a pool ready for immediate use. The process +of preparing the machines ahead of time reduces network traffic and +external dependencies during the run. -The mandate of the devstack-gate project is to prepare those virtual -machines, ensure that enough of them are always ready to run, -bootstrap the test process itself, and clean up when it's done. The -devstack gate scripts should be able to be configured to provision -machines based on several images (eg, natty, oneiric, precise), and -each of those from several providers. Using multiple providers makes -the entire system somewhat highly-available since only one provider -needs to function in order for us to run tests. Supporting multiple -images will help with the transition of testing from oneiric to -precise, and will allow us to continue running tests for stable -branches on older operating systems. +The `Nodepool`_ project is used to maintain this pool of machines. See -To accomplish all of that, the devstack-gate repository holds several -scripts that are run by Jenkins. - -Once per day, for every image type (and provider) configured, the -``devstack-vm-update-image.sh`` script checks out the latest copy of -devstack, and then runs the ``devstack-vm-update-image.py script.`` It -boots a new VM from the provider's base image, installs some basic -packages (build-essential, python-dev, etc) including java so that the -machine can run the Jenkins slave agent, runs puppet to set up the -basic system configuration for Jenkins slaves in the openstack-infra -project, and then caches all of the debian and pip packages and test -images specified in the devstack repository, and clones the OpenStack -project repositories. It then takes a snapshot image of that machine -to use when booting the actual test machines. When they boot, they -will already be configured and have all, or nearly all, of the network -accessible data they need. Then the template machine is deleted. The -Jenkins job that does this is ``devstack-update-vm-image``. It is a -matrix job that runs for all configured providers, and if any of them -fail, it's not a problem since the previously generated image will -still be available. - -Even though launching a machine from a saved image is usually fast, -depending on the provider's load it can sometimes take a while, and -it's possible that the resulting machine may end up in an error state, -or have some malfunction (such as a misconfigured network). Due to -these uncertainties, we provision the test machines ahead of time and -keep them in a pool. Every ten minutes, a job runs to spin up new VMs -for testing and add them to the pool, using the -``devstack-vm-launch.py`` script. Each image type has a parameter -specifying how many machine of that type should be kept ready, and -each provider has a parameter specifying the maximum number of -machines allowed to be running on that provider. Within those bounds, -the job attempts to keep the requested number of machines up and ready -to go at all times. When a machine is spun up and found to be -accessible, it as added to Jenkins as a slave machine with one -executor and a tag like "devstack-foo" (eg, "devstack-oneiric" for -oneiric image types). The Jenkins job that does this is -``devstack-launch-vms``. It is also a matrix job that runs for all -configured providers. - -Process invoked once a proposed change is approved by the core -reviewers is as follows: - - * Jenkins triggers the devstack gate test itself. - * This job runs on one of the previously configured "devstack-foo" - nodes and invokes the ``devstack-vm-gate-wrap.sh`` script which - checks out code from all of the involved repositories, and merges - the proposed change. - * If the ``pre_test_hook`` function is defined it is executed. - * The wrap script defines a ``gate_hook`` function if one is - not provided. By default it uses the devstack-vm-gate.sh script - which installs a devstack configuration file, and invokes devstack. - * If the ``post_test_hook`` function is defined it is executed. - * Once devstack is finished, it runs ``exercise.sh`` which performs - some basic integration testing. - * After everything is done, the script copies all of the log files - back to the Jenkins workspace and archives them along with the - console output of the run. The Jenkins job that does this is the - somewhat awkwardly named ``gate-integration-tests-devstack-vm``. - -To prevent a node from being used for a second run, there is a job -named ``devstack-update-inprogress`` which is triggered as a -parameterized build step from ``gate-interation-tests-devstack-vm``. -It is passed the name of the node on which the gate job is running, -and it disabled that node in Jenkins by invoking -``devstack-vm-inprogress.py``. The currently running job will -continue, but no new jobs will be scheduled for that node. - -Similarly, when the node is finished, a parameterized job named -``devstack-update-complete`` (which runs ``devstack-vm-delete.py``) -is triggered as a post-build action. It removes the node from Jenkins -and marks the VM for later deletion. - -In the future, we hope to be able to install developer SSH keys on VMs -from failed test runs, but for the moment the policies of the -providers who are donating test resources do not permit that. However, -most problems can be diagnosed from the log data that are copied back -to Jenkins. There is a script that cleans up old images and VMs that -runs frequently. It's ``devstack-vm-reap.py`` and is invoked by the -Jenkins job ``devstack-reap-vms``. +.. _Nodepool: https://git.openstack.org/cgit/openstack-infra/nodepool How to Debug a Devstack Gate Failure ==================================== @@ -230,7 +126,6 @@ line. A provider settings file for Rackspace would look something like:: export OS_TENANT_NAME= export OS_AUTH_URL=https://identity.api.rackspacecloud.com/v2.0/ export OS_REGION_NAME=DFW - export NOVA_RAX_AUTH=1 export FLAVOR='8GB Standard Instance' export IMAGE='Ubuntu 12.04 LTS (Precise Pangolin)' @@ -346,76 +241,12 @@ managed in source code repositories just like the code of OpenStack itself. If you'd like to contribute, just clone and propose a patch to the relevant repository:: - https://github.com/openstack-infra/devstack-gate - https://github.com/openstack/openstack-infra-puppet + https://git.openstack.org/cgit/openstack-infra/devstack-gate + https://git.openstack.org/cgit/openstack-infra/nodepool + https://git.openstack.org/cgit/openstack-infra/config You can file bugs on the openstack-ci project:: https://launchpad.net/openstack-ci And you can chat with us on Freenode in #openstack-dev or #openstack-infra. - -Developer Setup -=============== - -If you'd like to work on the devstack-gate scripts and test process, -this should help you bootstrap a test environment (assuming the user -you're working as is called "jenkins"):: - - export WORKSPACE=/home/jenkins/workspace - export DEVSTACK_GATE_PREFIX=wip- - export SKIP_DEVSTACK_GATE_PROJECT=1 - export SKIP_DEVSTACK_GATE_JENKINS=1 - export ZUUL_BRANCH=master - export ZUUL_PROJECT=testing - - cd /home/jenkins/workspace - git clone https://github.com/openstack-infra/devstack-gate - cd devstack-gate - python vmdatabase.py - sqlite3 /home/jenkins/vm.db - -With the database open, you'll want to populate the provider and base_image -tables with your provider details and specifications for images created. - -By default, the update-image script will produce a VM that only members -of the OpenStack CI team can log into. You can inject your SSH public -key by setting the appropriate env variable, like so:: - - export JENKINS_SSH_KEY=$(head -1 ~/.ssh/authorized_keys) - -Then run:: - - ./devstack-vm-update-image.sh - ./devstack-vm-launch.py - python vmdatabase.py - -So that you don't need an entire Jenkins environment during -development, The SKIP_DEVSTACK_GATE_JENKINS variable will cause the -launch and reap scripts to omit making changes to Jenkins. You'll -need to pick a machine to use yourself, so chose an IP from the output -from 'python vmdatabase.py' and then run:: - - ./devstack-vm-gate-dev.sh - -To test your changes. That script copies the workspace over to the -machine and invokes the gate script as Jenkins would. When you're -done, you'll need to run:: - - ./devstack-vm-reap.py --all-servers - -To clean up. - -Production Setup -================ - -In addition to the jobs described under "How It Works", you will need -to install a config file at ~/devstack-gate-secure.conf on the Jenkins -node where you are running the update-image, launch, and reap jobs -that looks like this:: - - [jenkins] - server=https://jenkins.example.com - user=jekins-user-with-admin-privs - apikey=1234567890abcdef1234567890abcdef - diff --git a/devstack-vm-check.py b/devstack-vm-check.py deleted file mode 100755 index 0e9c2188..00000000 --- a/devstack-vm-check.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python - -# Remove old devstack VMs that have been given to developers. - -# Copyright (C) 2011-2012 OpenStack LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import time -import getopt -import traceback -import ConfigParser - -import myjenkins -import vmdatabase -import utils - -PROVIDER_NAME = sys.argv[1] -DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG', - os.path.expanduser( - '~/devstack-gate-secure.conf')) -SKIP_DEVSTACK_GATE_JENKINS = os.environ.get('SKIP_DEVSTACK_GATE_JENKINS', None) - - -def check_machine(jenkins, machine): - utils.log.debug("Check ID: %s" % machine.id) - - try: - if utils.ssh_connect(machine.ip, 'jenkins'): - return - except: - utils.log.exception("Check failed ID: %s" % machine.id) - utils.log.debug("Set deleted ID: %s old state: %s" % ( - machine.id, machine.state)) - machine.state = vmdatabase.DELETE - if jenkins: - if machine.jenkins_name: - if jenkins.node_exists(machine.jenkins_name): - utils.log.debug("Delete jenkins node ID: %s" % machine.id) - jenkins.delete_node(machine.jenkins_name) - - machine.delete() - - -def main(): - db = vmdatabase.VMDatabase() - - if not SKIP_DEVSTACK_GATE_JENKINS: - config = ConfigParser.ConfigParser() - config.read(DEVSTACK_GATE_SECURE_CONFIG) - - jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'), - config.get('jenkins', 'user'), - config.get('jenkins', 'apikey')) - jenkins.get_info() - else: - jenkins = None - - provider = db.getProvider(PROVIDER_NAME) - print "Working with provider %s" % provider.name - - error = False - for machine in provider.machines: - if machine.state != vmdatabase.READY: - continue - print 'Checking machine', machine.name - try: - check_machine(jenkins, machine) - except: - error = True - traceback.print_exc() - - utils.update_stats(provider) - if error: - sys.exit(1) - - -if __name__ == '__main__': - main() diff --git a/devstack-vm-delete.py b/devstack-vm-delete.py deleted file mode 100755 index 9290e893..00000000 --- a/devstack-vm-delete.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -# Delete a devstack VM. - -# Copyright (C) 2011-2012 OpenStack LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import sys -from statsd import statsd -import traceback -import urllib - -import vmdatabase -import utils - -NODE_NAME = sys.argv[1] -UPSTREAM_BUILD_URL = os.environ.get('UPSTREAM_BUILD_URL', '') -UPSTREAM_JOB_NAME = os.environ.get('UPSTREAM_JOB_NAME', '') -UPSTREAM_BRANCH = os.environ.get('UPSTREAM_BRANCH', '') -BUILD_URL = os.environ.get('BUILD_URL', '') - - -def main(): - db = vmdatabase.VMDatabase() - - try: - machine = db.getMachineByJenkinsName(NODE_NAME) - except Exception: - utils.log.debug("Unable to find node: %s" % NODE_NAME) - return - - if machine.state != vmdatabase.HOLD: - utils.log.debug("Set deleted ID: %s old state: %s build: %s" % ( - machine.id, machine.state, BUILD_URL)) - machine.state = vmdatabase.DELETE - else: - utils.log.debug("Hold ID: %s old state: %s build: %s" % ( - machine.id, machine.state, BUILD_URL)) - - try: - utils.update_stats(machine.base_image.provider) - - if UPSTREAM_BUILD_URL: - fd = urllib.urlopen(UPSTREAM_BUILD_URL + 'api/json') - data = json.load(fd) - result = data['result'] - if statsd and result == 'SUCCESS': - dt = int(data['duration']) - - key = 'devstack.job.%s' % UPSTREAM_JOB_NAME - statsd.timing(key + '.runtime', dt) - statsd.incr(key + '.builds') - - key += '.%s' % UPSTREAM_BRANCH - statsd.timing(key + '.runtime', dt) - statsd.incr(key + '.builds') - - key += '.%s' % machine.base_image.provider.name - statsd.timing(key + '.runtime', dt) - statsd.incr(key + '.builds') - except: - print "Error getting build information" - traceback.print_exc() - -if __name__ == '__main__': - main() diff --git a/devstack-vm-fetch.py b/devstack-vm-fetch.py deleted file mode 100755 index 377a3119..00000000 --- a/devstack-vm-fetch.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python - -# Fetch a ready VM for use by devstack. - -# Copyright (C) 2011-2012 OpenStack LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import os - -import vmdatabase - -IMAGE_NAME = sys.argv[1] - - -def main(): - db = vmdatabase.VMDatabase() - node = db.getMachineForUse(IMAGE_NAME) - if not node: - raise Exception("No ready nodes") - - job_name = os.environ.get('JOB_NAME', None) - build_number = os.environ.get('BUILD_NUMBER', None) - gerrit_change = os.environ.get('GERRIT_CHANGE_NUMBER', None) - gerrit_patchset = os.environ.get('GERRIT_PATCHSET_NUMBER', None) - if job_name and build_number and gerrit_change and gerrit_patchset: - result = node.newResult(job_name, - build_number, - gerrit_change, - gerrit_patchset) - else: - result = None - - print "NODE_IP_ADDR=%s" % node.ip - print "NODE_PROVIDER=%s" % node.base_image.provider.name - print "NODE_ID=%s" % node.id - if result: - print "RESULT_ID=%s" % result.id - - -if __name__ == "__main__": - main() diff --git a/devstack-vm-give.py b/devstack-vm-give.py deleted file mode 100755 index 55d69dcf..00000000 --- a/devstack-vm-give.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python - -# Turn over a devstack configured machine to the developer who -# proposed the change that is being tested. - -# Copyright (C) 2011-2012 OpenStack LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import commands -import json -import urllib2 -import tempfile - -import vmdatabase - -NODE_ID = sys.argv[1] - - -def main(): - db = vmdatabase.VMDatabase() - machine = db.getMachine(NODE_ID) - - stat, out = commands.getstatusoutput( - "ssh -p 29418 review.openstack.org gerrit" + - "query --format=JSON change:%s" % - os.environ['GERRIT_CHANGE_NUMBER']) - - data = json.loads(out.split('\n')[0]) - username = data['owner']['username'] - - f = urllib2.urlopen('https://launchpad.net/~%s/+sshkeys' % username) - keys = f.read() - - tmp = tempfile.NamedTemporaryFile(delete=False) - try: - tmp.write("""#!/bin/bash -chmod u+w ~/.ssh/authorized_keys -cat <>~/.ssh/authorized_keys -""") - tmp.write(keys) - tmp.write("\nEOF\n") - tmp.close() - stat, out = commands.getstatusoutput("scp %s %s:/var/tmp/keys.sh" % - (tmp.name, machine.ip)) - if stat: - print out - raise Exception("Unable to copy keys") - - stat, out = commands.getstatusoutput( - "ssh %s /bin/sh /var/tmp/keys.sh" % machine.ip) - - if stat: - print out - raise Exception("Unable to add keys") - finally: - os.unlink(tmp.name) - - machine.user = username - print "Added %s to authorized_keys on %s" % (username, machine.ip) - -if __name__ == '__main__': - main() diff --git a/devstack-vm-inprogress.py b/devstack-vm-inprogress.py deleted file mode 100755 index bcbade52..00000000 --- a/devstack-vm-inprogress.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python - -# Remove old devstack VMs that have been given to developers. - -# Copyright (C) 2011-2012 OpenStack LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import ConfigParser - -import myjenkins -import vmdatabase -import utils -import re - -NODE_NAME = sys.argv[1] -DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG', - os.path.expanduser( - '~/devstack-gate-secure.conf')) -SKIP_DEVSTACK_GATE_JENKINS = os.environ.get('SKIP_DEVSTACK_GATE_JENKINS', None) -BUILD_URL = os.environ.get('BUILD_URL', '') - -LABEL_RE = re.compile(r'') - - -def main(): - db = vmdatabase.VMDatabase() - - config = ConfigParser.ConfigParser() - config.read(DEVSTACK_GATE_SECURE_CONFIG) - - if not SKIP_DEVSTACK_GATE_JENKINS: - jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'), - config.get('jenkins', 'user'), - config.get('jenkins', 'apikey')) - jenkins.get_info() - else: - jenkins = None - - try: - machine = db.getMachineByJenkinsName(NODE_NAME) - except Exception: - utils.log.debug("Unable to find node: %s" % NODE_NAME) - return - - utils.log.debug("Used ID: %s old state: %s build:%s" % ( - machine.id, machine.state, BUILD_URL)) - - machine.state = vmdatabase.USED - - if jenkins: - if machine.jenkins_name: - if jenkins.node_exists(machine.jenkins_name): - config = jenkins.get_node_config(machine.jenkins_name) - old = None - m = LABEL_RE.search(config) - if m: - old = m.group(1) - config = LABEL_RE.sub('', config) - for i in range(3): - try: - jenkins.reconfig_node(machine.jenkins_name, config) - except: - if i == 2: - utils.log.exception( - "Unable to relabel ID: %s" % machine.id) - raise - time.sleep(5) - utils.log.debug( - "Relabeled ID: %s old label: %s new label: %s" % ( - machine.id, old, 'devstack-used')) - - utils.update_stats(machine.base_image.provider) - -if __name__ == '__main__': - main() diff --git a/devstack-vm-launch.py b/devstack-vm-launch.py deleted file mode 100755 index 4dc04d61..00000000 --- a/devstack-vm-launch.py +++ /dev/null @@ -1,252 +0,0 @@ -#!/usr/bin/env python - -# Make sure there are always a certain number of VMs launched and -# ready for use by devstack. - -# Copyright (C) 2011-2012 OpenStack LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import time -import traceback -import ConfigParser -from statsd import statsd - -import myjenkins -import vmdatabase -import utils - -PROVIDER_NAME = sys.argv[1] -DEVSTACK_GATE_PREFIX = os.environ.get('DEVSTACK_GATE_PREFIX', '') -DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG', - os.path.expanduser( - '~/devstack-gate-secure.conf')) -SKIP_DEVSTACK_GATE_JENKINS = os.environ.get('SKIP_DEVSTACK_GATE_JENKINS', None) - -ABANDON_TIMEOUT = 900 # assume a machine will never boot if it hasn't - # after this amount of time - - -def calculate_deficit(provider, base_image): - # Count machines that are ready and machines that are building, - # so that if the provider is very slow, we aren't queueing up tons - # of machines to be built. - num_to_launch = base_image.min_ready - (len(base_image.ready_machines) + - len(base_image.building_machines)) - - # Don't launch more than our provider max - num_to_launch = min(provider.max_servers - len(provider.machines), - num_to_launch) - - # Don't launch less than 0 - num_to_launch = max(0, num_to_launch) - - print "Ready nodes: ", len(base_image.ready_machines) - print "Building nodes:", len(base_image.building_machines) - print "Provider total:", len(provider.machines) - print "Provider max: ", provider.max_servers - print "Need to launch:", num_to_launch - - return num_to_launch - - -def launch_node(client, snap_image, image, flavor, last_name): - while True: - name = '%sdevstack-%s.slave.openstack.org' % ( - DEVSTACK_GATE_PREFIX, int(time.time())) - if name != last_name: - break - time.sleep(1) - create_kwargs = dict(image=image, flavor=flavor, name=name) - server = client.servers.create(**create_kwargs) - machine = snap_image.base_image.newMachine(name=name, - external_id=server.id) - print "Started building machine %s:" % machine.id - print " id: %s" % (server.id) - print " name: %s" % (name) - print - utils.log.debug("Launching ID: %s name: %s provider ID: %s" % - (machine.id, name, server.id)) - return server, machine - - -def create_jenkins_node(jenkins, machine, credentials_id): - name = '%sdevstack-%s-%s-%s' % (DEVSTACK_GATE_PREFIX, - machine.base_image.name, - machine.base_image.provider.name, - machine.id) - machine.jenkins_name = name - - if jenkins: - node_desc = 'Dynamic single use %s slave for devstack' % \ - machine.base_image.name - labels = '%sdevstack-%s' % (DEVSTACK_GATE_PREFIX, - machine.base_image.name) - priv_key = '/var/lib/jenkins/.ssh/id_rsa' - if credentials_id: - launcher_params={'port': 22, - 'credentialsId': credentials_id, - 'host': machine.ip} - else: - launcher_params={'port': 22, - 'username': 'jenkins', - 'privatekey': priv_key, - 'host': machine.ip} - - try: - jenkins.create_node( - name, numExecutors=1, - nodeDescription=node_desc, - remoteFS='/home/jenkins', - labels=labels, - exclusive=True, - launcher='hudson.plugins.sshslaves.SSHLauncher', - launcher_params=launcher_params) - except myjenkins.JenkinsException as e: - if 'already exists' in str(e): - pass - else: - raise - - -def check_machine(jenkins, client, machine, error_counts, credentials_id): - try: - server = client.servers.get(machine.external_id) - except: - print "Unable to get server detail, will retry" - traceback.print_exc() - return - - if server.status == 'ACTIVE': - ip = utils.get_public_ip(server) - if not ip and 'os-floating-ips' in utils.get_extensions(client): - utils.add_public_ip(server) - ip = utils.get_public_ip(server) - if not ip: - raise Exception("Unable to find public ip of server") - - machine.ip = ip - print "Machine %s is running, testing ssh" % machine.id - if utils.ssh_connect(ip, 'jenkins'): - if statsd: - dt = int((time.time() - machine.state_time) * 1000) - key = 'devstack.launch.%s' % machine.base_image.provider.name - statsd.timing(key, dt) - statsd.incr(key) - print "Adding machine %s to Jenkins" % machine.id - create_jenkins_node(jenkins, machine, credentials_id) - print "Machine %s is ready" % machine.id - machine.state = vmdatabase.READY - utils.log.debug("Online ID: %s" % machine.id) - return - elif not server.status.startswith('BUILD'): - count = error_counts.get(machine.id, 0) - count += 1 - error_counts[machine.id] = count - print "Machine %s is in error %s (%s/5)" % (machine.id, - server.status, - count) - if count >= 5: - if statsd: - statsd.incr('devstack.error.%s' % - machine.base_image.provider.name) - raise Exception("Too many errors querying machine %s" % machine.id) - else: - if time.time() - machine.state_time >= ABANDON_TIMEOUT: - if statsd: - statsd.incr('devstack.timeout.%s' % - machine.base_image.provider.name) - raise Exception("Waited too long for machine %s" % machine.id) - - -def main(): - db = vmdatabase.VMDatabase() - - jenkins = None - credentials_id = None - if not SKIP_DEVSTACK_GATE_JENKINS: - config = ConfigParser.ConfigParser() - config.read(DEVSTACK_GATE_SECURE_CONFIG) - - jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'), - config.get('jenkins', 'user'), - config.get('jenkins', 'apikey')) - jenkins.get_info() - if config.has_option('jenkins', 'credentials_id'): - credentials_id = config.get('jenkins', 'credentials_id') - - provider = db.getProvider(PROVIDER_NAME) - print "Working with provider %s" % provider.name - - client = utils.get_client(provider) - - last_name = '' - error_counts = {} - error = False - - for base_image in provider.base_images: - snap_image = base_image.current_snapshot - if not snap_image: - continue - print "Working on image %s" % snap_image.name - - flavor = utils.get_flavor(client, base_image.min_ram) - print "Found flavor", flavor - - remote_snap_image = client.images.get(snap_image.external_id) - print "Found image", remote_snap_image - - num_to_launch = calculate_deficit(provider, base_image) - for i in range(num_to_launch): - try: - server, machine = launch_node(client, - snap_image, - remote_snap_image, - flavor, - last_name) - last_name = machine.name - except: - traceback.print_exc() - error = True - - while True: - utils.update_stats(provider) - building_machines = provider.building_machines - if not building_machines: - print "No more machines are building, finished." - break - - print "Waiting on %s machines" % len(building_machines) - for machine in building_machines: - try: - check_machine(jenkins, client, machine, error_counts, credentials_id) - except: - traceback.print_exc() - print "Abandoning machine %s" % machine.id - utils.log.exception("Abandoning ID: %s" % machine.id) - machine.state = vmdatabase.ERROR - error = True - db.commit() - - time.sleep(3) - - if error: - sys.exit(1) - - -if __name__ == '__main__': - main() diff --git a/devstack-vm-reap.py b/devstack-vm-reap.py deleted file mode 100755 index d1316a99..00000000 --- a/devstack-vm-reap.py +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env python - -# Remove old devstack VMs that have been given to developers. - -# Copyright (C) 2011-2012 OpenStack LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import time -import traceback -import ConfigParser - -import myjenkins -import vmdatabase -import utils -import novaclient - -PROVIDER_NAME = sys.argv[1] -MACHINE_LIFETIME = 24 * 60 * 60 # Amount of time after being used -DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG', - os.path.expanduser( - '~/devstack-gate-secure.conf')) -SKIP_DEVSTACK_GATE_JENKINS = os.environ.get('SKIP_DEVSTACK_GATE_JENKINS', None) - -if '--all-servers' in sys.argv: - print "Reaping all known machines" - REAP_ALL_SERVERS = True -else: - REAP_ALL_SERVERS = False - -if '--all-images' in sys.argv: - print "Reaping all known images" - REAP_ALL_IMAGES = True -else: - REAP_ALL_IMAGES = False - - -def delete_machine(jenkins, client, machine): - if machine.state != vmdatabase.DELETE: - utils.log.debug("Set deleted ID: %s old state: %s" % ( - machine.id, machine.state)) - machine.state = vmdatabase.DELETE - - try: - server = client.servers.get(machine.external_id) - except novaclient.exceptions.NotFound: - print ' Machine id %s not found' % machine.external_id - server = None - - if server: - utils.delete_server(server) - # Rackspace Nova sometimes lies about whether a server is deleted. - # If we have deleted a server, don't believe it. Instead, wait for - # the next run of the script and only if the server doesn't exist, - # delete it from Jenkins and the DB. - utils.log.debug("Delete ID: %s" % machine.id) - return - - if jenkins: - if machine.jenkins_name: - if jenkins.node_exists(machine.jenkins_name): - jenkins.delete_node(machine.jenkins_name) - utils.log.debug("Delete jenkins node ID: %s" % machine.id) - - machine.delete() - - -def delete_image(client, image): - try: - server = client.servers.get(image.server_external_id) - except novaclient.exceptions.NotFound: - print ' Image server id %s not found' % image.server_external_id - server = None - - if server: - utils.delete_server(server) - # Rackspace Nova sometimes lies about whether a server is deleted. - # If we have deleted a server, don't believe it. Instead, wait for - # the next run of the script and only if the server doesn't exist, - # delete it from Jenkins and the DB. - return - - try: - remote_image = client.images.get(image.external_id) - except novaclient.exceptions.NotFound: - print ' Image id %s not found' % image.external_id - remote_image = None - - if remote_image: - remote_image.delete() - - image.delete() - - -def main(): - db = vmdatabase.VMDatabase() - - if not SKIP_DEVSTACK_GATE_JENKINS: - config = ConfigParser.ConfigParser() - config.read(DEVSTACK_GATE_SECURE_CONFIG) - - jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'), - config.get('jenkins', 'user'), - config.get('jenkins', 'apikey')) - jenkins.get_info() - else: - jenkins = None - - print 'Known machines (start):' - db.print_state() - - provider = db.getProvider(PROVIDER_NAME) - print "Working with provider %s" % provider.name - - client = utils.get_client(provider) - - flavor = utils.get_flavor(client, 1024) - print "Found flavor", flavor - - error = False - now = time.time() - for machine in provider.machines: - # Normally, reap machines that have sat in their current state - # for 24 hours, unless that state is READY. - if (REAP_ALL_SERVERS or - (machine.state != vmdatabase.READY and - now - machine.state_time > MACHINE_LIFETIME) or - machine.state == vmdatabase.DELETE or - machine.state == vmdatabase.ERROR): - print 'Deleting machine', machine.name - try: - delete_machine(jenkins, client, machine) - except: - error = True - traceback.print_exc() - - provider_min_ready = 0 - for base_image in provider.base_images: - provider_min_ready += base_image.min_ready - for snap_image in base_image.snapshot_images: - # Normally, reap images that have sat in their current state - # for 24 hours, unless the image is the current snapshot - if (REAP_ALL_IMAGES or - (snap_image != base_image.current_snapshot and - now - snap_image.state_time > MACHINE_LIFETIME)): - print 'Deleting image', snap_image.name - try: - delete_image(client, snap_image) - except: - error = True - traceback.print_exc() - - print - print 'Known machines (end):' - db.print_state() - - utils.update_stats(provider) - if error: - sys.exit(1) - - -if __name__ == '__main__': - main() diff --git a/devstack-vm-result.py b/devstack-vm-result.py deleted file mode 100755 index f09d4d98..00000000 --- a/devstack-vm-result.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python - -# Record a result from a build in the database. - -# Copyright (C) 2011-2012 OpenStack LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys - -import vmdatabase - -RESULT_ID = sys.argv[1] -RESULT = sys.argv[2] - -RESULTS = dict(success=vmdatabase.RESULT_SUCCESS, - failure=vmdatabase.RESULT_FAILURE, - timeout=vmdatabase.RESULT_TIMEOUT, - ) - - -def main(): - db = vmdatabase.VMDatabase() - result = db.getResult(RESULT_ID) - - value = RESULTS[RESULT] - # This gets called with an argument of 'timeout' after every run, - # regardless of whether a timeout occured; so in that case, only - # set the result to timeout if there is not already a result. - if not (value == vmdatabase.RESULT_TIMEOUT and result.result): - result.setResult(value) - -if __name__ == '__main__': - main() diff --git a/devstack-vm-threshold.py b/devstack-vm-threshold.py deleted file mode 100755 index 83c9c812..00000000 --- a/devstack-vm-threshold.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python - -# Get count of available slaves. - -# Copyright (C) 2011 OpenStack LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -import vmdatabase -import os -import sys -import getopt - - -def main(threshold, stat_file): - db = vmdatabase.VMDatabase() - ready = vmdatabase.READY - ready_nodes = [] - - for provider in db.getProviders(): - for base_image in provider.base_images: - ready_nodes = [x for x in base_image.machines if x.state == ready] - ready_count = len(ready_nodes) - set_vm_state(ready_count, stat_file) - - print "Number of slaves available: %s\n" % ready_count - if ready_count < threshold: - sys.exit(1) - - -def set_vm_state(count, stat_file): - try: - w_fh = open(stat_file, 'w') - w_fh.write("slaves\n%s\n" % count) - w_fh.close() - except IOError, err: - print >>sys.stderr, "warning: unable to update stat file: %s" % err - - -def usage(msg=None): - if msg: - stream = sys.stderr - else: - stream = sys.stdout - stream.write("usage: %s [-h] -t threshold [-f stat-file]\n" - % os.path.basename(sys.argv[0])) - if msg: - stream.write("\nERROR: " + msg + "\n") - exitCode = 1 - else: - exitCode = 0 - sys.exit(exitCode) - -if __name__ == '__main__': - try: - opts = getopt.getopt(sys.argv[1:], 'ht:f:')[0] - except getopt.GetoptError: - usage('invalid option selected') - - threshold = None - stat_file = None - for opt, value in opts: - if (opt in ('-h')): - usage() - elif (opt in ('-f')): - stat_file = value - elif (opt in ('-t')): - threshold = value - - if not threshold: - usage('please specify threshold') - try: - threshold = int(threshold) - except TypeError, err: - usage('invalid threshold specified') - - if not stat_file: - stat_file = os.path.expanduser('~/vm-threshold.txt') - - main(threshold, stat_file) diff --git a/devstack-vm-update-image.py b/devstack-vm-update-image.py deleted file mode 100755 index 7fdcdb64..00000000 --- a/devstack-vm-update-image.py +++ /dev/null @@ -1,367 +0,0 @@ -#!/usr/bin/env python - -# Update the base image that is used for devstack VMs. - -# Copyright (C) 2011-2012 OpenStack LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - - -import sys -import os -import time -import subprocess -import traceback -import pprint - -import vmdatabase -import utils -from sshclient import SSHClient - -WORKSPACE = os.environ['WORKSPACE'] -DEVSTACK_GATE_PREFIX = os.environ.get('DEVSTACK_GATE_PREFIX', '') -DEVSTACK = os.path.join(WORKSPACE, 'devstack') -PROVIDER_NAME = sys.argv[1] -JENKINS_SSH_KEY = os.environ.get('JENKINS_SSH_KEY', False) - -if JENKINS_SSH_KEY: - PUPPET_CLASS = ("class {'openstack_project::slave_template': " - "install_users => false, ssh_key => '%s', }" % - JENKINS_SSH_KEY) -else: - PUPPET_CLASS = "class {'openstack_project::slave_template': }" - -PROJECTS = [ - 'openstack-dev/devstack', - 'openstack-dev/grenade', - 'openstack-dev/pbr', - 'openstack-infra/devstack-gate', - 'openstack-infra/jeepyb', - 'openstack/ceilometer', - 'openstack/cinder', - 'openstack/glance', - 'openstack/heat', - 'openstack/horizon', - 'openstack/keystone', - 'openstack/neutron', - 'openstack/nova', - 'openstack/oslo.config', - 'openstack/oslo.messaging', - 'openstack/python-ceilometerclient', - 'openstack/python-cinderclient', - 'openstack/python-glanceclient', - 'openstack/python-heatclient', - 'openstack/python-keystoneclient', - 'openstack/python-neutronclient', - 'openstack/python-novaclient', - 'openstack/python-openstackclient', - 'openstack/python-swiftclient', - 'openstack/requirements', - 'openstack/swift', - 'openstack/tempest', -] - - -def run_local(cmd, status=False, cwd='.', env={}): - print "Running:", cmd - newenv = os.environ - newenv.update(env) - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd, - stderr=subprocess.STDOUT, env=newenv) - (out, nothing) = p.communicate() - if status: - return (p.returncode, out.strip()) - return out.strip() - - -def git_branches(): - branches = [] - for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"): - branch = branch.strip() - if not branch.startswith('remotes/origin'): - continue - branches.append(branch) - return branches - - -def tokenize(fn, tokens, distribution, comment=None): - for line in open(fn): - if 'dist:' in line and ('dist:%s' % distribution not in line): - continue - if 'qpid' in line: - continue # TODO: explain why this is here - if comment and comment in line: - line = line[:line.rfind(comment)] - line = line.strip() - if line and line not in tokens: - tokens.append(line) - - -def local_prep(distribution): - branches = [] - for branch in git_branches(): - # Ignore branches of the form 'somestring -> someotherstring' as - # this denotes a symbolic reference and the entire string as is - # cannot be checkout out. We can do this safely as the reference - # will refer to one of the other branches returned by git_branches. - if ' -> ' in branch: - continue - branch_data = {'name': branch} - print 'Branch: ', branch - run_local(['git', 'checkout', branch], cwd=DEVSTACK) - run_local(['git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK) - - pips = [] - pipdir = os.path.join(DEVSTACK, 'files', 'pips') - if os.path.exists(pipdir): - for fn in os.listdir(pipdir): - fn = os.path.join(pipdir, fn) - tokenize(fn, pips, distribution) - branch_data['pips'] = pips - - debs = [] - debdir = os.path.join(DEVSTACK, 'files', 'apts') - for fn in os.listdir(debdir): - fn = os.path.join(debdir, fn) - tokenize(fn, debs, distribution, comment='#') - branch_data['debs'] = debs - - images = [] - for line in open(os.path.join(DEVSTACK, 'stackrc')): - line = line.strip() - if line.startswith('IMAGE_URLS'): - if '#' in line: - line = line[:line.rfind('#')] - if line.endswith(';;'): - line = line[:-2] - line = line.split('=', 1)[1].strip() - if line.startswith('${IMAGE_URLS:-'): - line = line[len('${IMAGE_URLS:-'):] - if line.endswith('}'): - line = line[:-1] - if line[0] == line[-1] == '"': - line = line[1:-1] - images += [x.strip() for x in line.split(',')] - branch_data['images'] = images - branches.append(branch_data) - return branches - - -def bootstrap_server(provider, server, admin_pass, key): - client = server.manager.api - ip = utils.get_public_ip(server) - if not ip and 'os-floating-ips' in utils.get_extensions(client): - utils.add_public_ip(server) - ip = utils.get_public_ip(server) - if not ip: - raise Exception("Unable to find public ip of server") - - ssh_kwargs = {} - if key: - ssh_kwargs['pkey'] = key - else: - ssh_kwargs['password'] = admin_pass - - for username in ['root', 'ubuntu']: - client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) - if client: - break - - if not client: - raise Exception("Unable to log in via SSH") - - # hpcloud can't reliably set the hostname - gerrit_url = 'https://review.openstack.org/p/openstack-infra/config.git' - client.ssh("set hostname", "sudo hostname %s" % server.name) - client.ssh("get puppet repo deb", - "sudo /usr/bin/wget " - "http://apt.puppetlabs.com/puppetlabs-release-" - "`lsb_release -c -s`.deb -O /root/puppet-repo.deb") - client.ssh("install puppet repo deb", "sudo dpkg -i /root/puppet-repo.deb") - client.ssh("update apt cache", "sudo apt-get update") - client.ssh("upgrading system packages", - 'sudo DEBIAN_FRONTEND=noninteractive apt-get ' - '--option "Dpkg::Options::=--force-confold"' - ' --assume-yes dist-upgrade') - client.ssh("install git and puppet", - 'sudo DEBIAN_FRONTEND=noninteractive apt-get ' - '--option "Dpkg::Options::=--force-confold"' - ' --assume-yes install git puppet') - client.ssh("clone puppret repo", - "sudo git clone %s /root/config" % gerrit_url) - client.ssh("install puppet modules", - "sudo /bin/bash /root/config/install_modules.sh") - client.ssh("run puppet", - "sudo puppet apply --modulepath=/root/config/modules:" - "/etc/puppet/modules " - '-e "%s"' % PUPPET_CLASS) - - -def configure_server(server, branches): - client = SSHClient(utils.get_public_ip(server), 'jenkins') - client.ssh('make file cache directory', 'mkdir -p ~/cache/files') - client.ssh('make pip cache directory', 'mkdir -p ~/cache/pip') - client.ssh('install build-essential', - 'sudo DEBIAN_FRONTEND=noninteractive ' - 'apt-get --option "Dpkg::Options::=--force-confold"' - ' --assume-yes install build-essential python-dev ' - 'linux-headers-virtual linux-headers-`uname -r`') - - for branch_data in branches: - if branch_data['debs']: - client.ssh('cache debs for branch %s' % branch_data['name'], - 'sudo apt-get -y -d install %s' % - ' '.join(branch_data['debs'])) - - if branch_data['pips']: - venv = client.ssh('get temp dir for venv', 'mktemp -d').strip() - client.ssh('create venv', - 'virtualenv --no-site-packages %s' % venv) - client.ssh('cache pips for branch %s' % branch_data['name'], - 'source %s/bin/activate && ' - 'PIP_DOWNLOAD_CACHE=~/cache/pip pip install %s' % - (venv, ' '.join(branch_data['pips']))) - client.ssh('remove venv', 'rm -fr %s' % venv) - - for url in branch_data['images']: - fname = url.split('/')[-1] - try: - client.ssh('check for %s' % fname, - 'ls ~/cache/files/%s' % fname) - except: - client.ssh('download image %s' % fname, - 'wget -nv -c %s -O ~/cache/files/%s' % (url, fname)) - - client.ssh('clear workspace', 'rm -rf ~/workspace-cache') - client.ssh('make workspace', 'mkdir -p ~/workspace-cache') - for project in PROJECTS: - client.ssh('clone %s' % project, - 'cd ~/workspace-cache && ' - 'git clone https://review.openstack.org/p/%s' % project) - - script = os.environ.get('DEVSTACK_GATE_CUSTOM_SCRIPT', '') - if script and os.path.isfile(script): - bn = os.path.basename(script) - client.scp(script, '/tmp/%s' % bn) - client.ssh('run custom script %s' % bn, - 'chmod +x /tmp/%s && sudo /tmp/%s' % (bn, bn)) - - client.ssh('sync', 'sync && sleep 5') - - -def snapshot_server(client, server, name): - print 'Saving image' - if hasattr(client.images, 'create'): # v1.0 - image = client.images.create(server, name) - else: - # TODO: fix novaclient so it returns an image here - # image = server.create_image(name) - uuid = server.manager.create_image(server, name) - image = client.images.get(uuid) - print "Waiting for image ID %s" % image.id - image = utils.wait_for_resource(image) - return image - - -def build_image(provider, client, base_image, image, - flavor, name, branches, timestamp): - print "Building image %s" % name - - create_kwargs = dict(image=image, flavor=flavor, name=name) - - key = None - key_name = '%sdevstack-%i' % (DEVSTACK_GATE_PREFIX, time.time()) - if 'os-keypairs' in utils.get_extensions(client): - print "Adding keypair" - key, kp = utils.add_keypair(client, key_name) - create_kwargs['key_name'] = key_name - - server = client.servers.create(**create_kwargs) - snap_image = base_image.newSnapshotImage(name=name, - version=timestamp, - external_id=None, - server_external_id=server.id) - admin_pass = server.adminPass - try: - print "Waiting for server ID %s" % server.id - server = utils.wait_for_resource(server) - bootstrap_server(provider, server, admin_pass, key) - configure_server(server, branches) - remote_snap_image = snapshot_server(client, server, name) - snap_image.external_id = remote_snap_image.id - snap_image.state = vmdatabase.READY - # We made the snapshot, try deleting the server, but it's okay - # if we fail. The reap script will find it and try again. - try: - utils.delete_server(server) - except: - print "Exception encountered deleting server:" - traceback.print_exc() - except Exception: - # Something went wrong, try our best to mark the server in error - # then delete the server, then delete the db record for it. - # If any of this fails, the reap script should catch it. But - # having correct info in the DB will help it do its job faster. - try: - snap_image.state = vmdatabase.ERROR - try: - utils.delete_server(server) - snap_image.delete() - except Exception: - print "Exception encountered deleting server:" - traceback.print_exc() - except Exception: - print "Exception encountered marking server in error:" - traceback.print_exc() - # Raise the important exception that started this - raise - - -def main(): - if '-n' in sys.argv: - dry = True - else: - dry = False - - db = vmdatabase.VMDatabase() - provider = db.getProvider(PROVIDER_NAME) - print "Working with provider %s" % provider.name - client = utils.get_client(provider) - - for base_image in provider.base_images: - if base_image.min_ready < 0: - continue - print "Working on base image %s" % base_image.name - - flavor = utils.get_flavor(client, base_image.min_ram) - print "Found flavor", flavor - - branches = local_prep(base_image.name) - pprint.pprint(branches) - - remote_base_image = client.images.find(name=base_image.external_id) - if not dry: - timestamp = int(time.time()) - remote_snap_image_name = ( - '%sdevstack-%s-%s.template.openstack.org' % - (DEVSTACK_GATE_PREFIX, base_image.name, str(timestamp))) - build_image(provider, client, base_image, - remote_base_image, flavor, - remote_snap_image_name, - branches, timestamp) - - -if __name__ == '__main__': - main() diff --git a/devstack-vm-update-image.sh b/devstack-vm-update-image.sh deleted file mode 100755 index 734392d7..00000000 --- a/devstack-vm-update-image.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -xe - -# Update the VM used in devstack deployments. - -# Copyright (C) 2011-2012 OpenStack LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -GATE_SCRIPT_DIR=$(cd $(dirname "$0") && pwd) -cd $WORKSPACE - -if [[ ! -e devstack ]]; then - git clone https://review.openstack.org/p/openstack-dev/devstack -fi -cd devstack -git remote update -git remote prune origin -cd $WORKSPACE - -$GATE_SCRIPT_DIR/devstack-vm-update-image.py $1 $2 - diff --git a/devstackgate/__init__.py b/devstackgate/__init__.py deleted file mode 100644 index 0a3b9886..00000000 --- a/devstackgate/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2011 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. diff --git a/devstackgate/openstack/__init__.py b/devstackgate/openstack/__init__.py deleted file mode 100644 index 0a3b9886..00000000 --- a/devstackgate/openstack/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2011 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. diff --git a/devstackgate/openstack/common/__init__.py b/devstackgate/openstack/common/__init__.py deleted file mode 100644 index 0a3b9886..00000000 --- a/devstackgate/openstack/common/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2011 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. diff --git a/devstackgate/openstack/common/setup.py b/devstackgate/openstack/common/setup.py deleted file mode 100644 index 59f255d0..00000000 --- a/devstackgate/openstack/common/setup.py +++ /dev/null @@ -1,359 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Utilities with minimum-depends for use in setup.py -""" - -import datetime -import os -import re -import subprocess -import sys - -from setuptools.command import sdist - - -def parse_mailmap(mailmap='.mailmap'): - mapping = {} - if os.path.exists(mailmap): - fp = open(mailmap, 'r') - for l in fp: - l = l.strip() - if not l.startswith('#') and ' ' in l: - canonical_email, alias = [x for x in l.split(' ') - if x.startswith('<')] - mapping[alias] = canonical_email - return mapping - - -def canonicalize_emails(changelog, mapping): - """Takes in a string and an email alias mapping and replaces all - instances of the aliases in the string with their real email. - """ - for alias, email in mapping.iteritems(): - changelog = changelog.replace(alias, email) - return changelog - - -# Get requirements from the first file that exists -def get_reqs_from_files(requirements_files): - reqs_in = [] - for requirements_file in requirements_files: - if os.path.exists(requirements_file): - return open(requirements_file, 'r').read().split('\n') - return [] - - -def parse_requirements(requirements_files=['requirements.txt', - 'tools/pip-requires']): - requirements = [] - for line in get_reqs_from_files(requirements_files): - # For the requirements list, we need to inject only the portion - # after egg= so that distutils knows the package it's looking for - # such as: - # -e git://github.com/openstack/nova/master#egg=nova - if re.match(r'\s*-e\s+', line): - requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', - line)) - # such as: - # http://github.com/openstack/nova/zipball/master#egg=nova - elif re.match(r'\s*https?:', line): - requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1', - line)) - # -f lines are for index locations, and don't get used here - elif re.match(r'\s*-f\s+', line): - pass - # argparse is part of the standard library starting with 2.7 - # adding it to the requirements list screws distro installs - elif line == 'argparse' and sys.version_info >= (2, 7): - pass - else: - requirements.append(line) - - return requirements - - -def parse_dependency_links(requirements_files=['requirements.txt', - 'tools/pip-requires']): - dependency_links = [] - # dependency_links inject alternate locations to find packages listed - # in requirements - for line in get_reqs_from_files(requirements_files): - # skip comments and blank lines - if re.match(r'(\s*#)|(\s*$)', line): - continue - # lines with -e or -f need the whole line, minus the flag - if re.match(r'\s*-[ef]\s+', line): - dependency_links.append(re.sub(r'\s*-[ef]\s+', '', line)) - # lines that are only urls can go in unmolested - elif re.match(r'\s*https?:', line): - dependency_links.append(line) - return dependency_links - - -def write_requirements(): - venv = os.environ.get('VIRTUAL_ENV', None) - if venv is not None: - with open("requirements.txt", "w") as req_file: - output = subprocess.Popen(["pip", "-E", venv, "freeze", "-l"], - stdout=subprocess.PIPE) - requirements = output.communicate()[0].strip() - req_file.write(requirements) - - -def _run_shell_command(cmd): - output = subprocess.Popen(["/bin/sh", "-c", cmd], - stdout=subprocess.PIPE) - out = output.communicate() - if len(out) == 0: - return None - if len(out[0].strip()) == 0: - return None - return out[0].strip() - - -def _get_git_next_version_suffix(branch_name): - datestamp = datetime.datetime.now().strftime('%Y%m%d') - if branch_name == 'milestone-proposed': - revno_prefix = "r" - else: - revno_prefix = "" - _run_shell_command("git fetch origin +refs/meta/*:refs/remotes/meta/*") - milestone_cmd = "git show meta/openstack/release:%s" % branch_name - milestonever = _run_shell_command(milestone_cmd) - if not milestonever: - milestonever = "" - post_version = _get_git_post_version() - # post version should look like: - # 0.1.1.4.gcc9e28a - # where the bit after the last . is the short sha, and the bit between - # the last and second to last is the revno count - (revno, sha) = post_version.split(".")[-2:] - first_half = "%(milestonever)s~%(datestamp)s" % locals() - second_half = "%(revno_prefix)s%(revno)s.%(sha)s" % locals() - return ".".join((first_half, second_half)) - - -def _get_git_current_tag(): - return _run_shell_command("git tag --contains HEAD") - - -def _get_git_tag_info(): - return _run_shell_command("git describe --tags") - - -def _get_git_post_version(): - current_tag = _get_git_current_tag() - if current_tag is not None: - return current_tag - else: - tag_info = _get_git_tag_info() - if tag_info is None: - base_version = "0.0" - cmd = "git --no-pager log --oneline" - out = _run_shell_command(cmd) - revno = len(out.split("\n")) - sha = _run_shell_command("git describe --always") - else: - tag_infos = tag_info.split("-") - base_version = "-".join(tag_infos[:-2]) - (revno, sha) = tag_infos[-2:] - return "%s.%s.%s" % (base_version, revno, sha) - - -def write_git_changelog(): - """Write a changelog based on the git changelog.""" - new_changelog = 'ChangeLog' - if not os.getenv('SKIP_WRITE_GIT_CHANGELOG'): - if os.path.isdir('.git'): - git_log_cmd = 'git log --stat' - changelog = _run_shell_command(git_log_cmd) - mailmap = parse_mailmap() - with open(new_changelog, "w") as changelog_file: - changelog_file.write(canonicalize_emails(changelog, mailmap)) - else: - open(new_changelog, 'w').close() - - -def generate_authors(): - """Create AUTHORS file using git commits.""" - jenkins_email = 'jenkins@review.openstack.org' - old_authors = 'AUTHORS.in' - new_authors = 'AUTHORS' - if not os.getenv('SKIP_GENERATE_AUTHORS'): - if os.path.isdir('.git'): - # don't include jenkins email address in AUTHORS file - git_log_cmd = ("git log --format='%aN <%aE>' | sort -u | " - "grep -v " + jenkins_email) - changelog = _run_shell_command(git_log_cmd) - mailmap = parse_mailmap() - with open(new_authors, 'w') as new_authors_fh: - new_authors_fh.write(canonicalize_emails(changelog, mailmap)) - if os.path.exists(old_authors): - with open(old_authors, "r") as old_authors_fh: - new_authors_fh.write('\n' + old_authors_fh.read()) - else: - open(new_authors, 'w').close() - - -_rst_template = """%(heading)s -%(underline)s - -.. automodule:: %(module)s - :members: - :undoc-members: - :show-inheritance: -""" - - -def read_versioninfo(project): - """Read the versioninfo file. If it doesn't exist, we're in a github - zipball, and there's really no way to know what version we really - are, but that should be ok, because the utility of that should be - just about nil if this code path is in use in the first place.""" - versioninfo_path = os.path.join(project, 'versioninfo') - if os.path.exists(versioninfo_path): - with open(versioninfo_path, 'r') as vinfo: - version = vinfo.read().strip() - else: - version = "0.0.0" - return version - - -def write_versioninfo(project, version): - """Write a simple file containing the version of the package.""" - open(os.path.join(project, 'versioninfo'), 'w').write("%s\n" % version) - - -def get_cmdclass(): - """Return dict of commands to run from setup.py.""" - - cmdclass = dict() - - def _find_modules(arg, dirname, files): - for filename in files: - if filename.endswith('.py') and filename != '__init__.py': - arg["%s.%s" % (dirname.replace('/', '.'), - filename[:-3])] = True - - class LocalSDist(sdist.sdist): - """Builds the ChangeLog and Authors files from VC first.""" - - def run(self): - write_git_changelog() - generate_authors() - # sdist.sdist is an old style class, can't use super() - sdist.sdist.run(self) - - cmdclass['sdist'] = LocalSDist - - # If Sphinx is installed on the box running setup.py, - # enable setup.py to build the documentation, otherwise, - # just ignore it - try: - from sphinx.setup_command import BuildDoc - - class LocalBuildDoc(BuildDoc): - def generate_autoindex(self): - print "**Autodocumenting from %s" % os.path.abspath(os.curdir) - modules = {} - option_dict = self.distribution.get_option_dict('build_sphinx') - source_dir = os.path.join(option_dict['source_dir'][1], 'api') - if not os.path.exists(source_dir): - os.makedirs(source_dir) - for pkg in self.distribution.packages: - if '.' not in pkg: - os.path.walk(pkg, _find_modules, modules) - module_list = modules.keys() - module_list.sort() - autoindex_filename = os.path.join(source_dir, 'autoindex.rst') - with open(autoindex_filename, 'w') as autoindex: - autoindex.write(""".. toctree:: - :maxdepth: 1 - -""") - for module in module_list: - output_filename = os.path.join(source_dir, - "%s.rst" % module) - heading = "The :mod:`%s` Module" % module - underline = "=" * len(heading) - values = dict(module=module, heading=heading, - underline=underline) - - print "Generating %s" % output_filename - with open(output_filename, 'w') as output_file: - output_file.write(_rst_template % values) - autoindex.write(" %s.rst\n" % module) - - def run(self): - if not os.getenv('SPHINX_DEBUG'): - self.generate_autoindex() - - for builder in ['html', 'man']: - self.builder = builder - self.finalize_options() - self.project = self.distribution.get_name() - self.version = self.distribution.get_version() - self.release = self.distribution.get_version() - BuildDoc.run(self) - cmdclass['build_sphinx'] = LocalBuildDoc - except ImportError: - pass - - return cmdclass - - -def get_git_branchname(): - for branch in _run_shell_command("git branch --color=never").split("\n"): - if branch.startswith('*'): - _branch_name = branch.split()[1].strip() - if _branch_name == "(no": - _branch_name = "no-branch" - return _branch_name - - -def get_pre_version(projectname, base_version): - """Return a version which is leading up to a version that will - be released in the future.""" - if os.path.isdir('.git'): - current_tag = _get_git_current_tag() - if current_tag is not None: - version = current_tag - else: - branch_name = os.getenv('BRANCHNAME', - os.getenv('GERRIT_REFNAME', - get_git_branchname())) - version_suffix = _get_git_next_version_suffix(branch_name) - version = "%s~%s" % (base_version, version_suffix) - write_versioninfo(projectname, version) - return version - else: - version = read_versioninfo(projectname) - return version - - -def get_post_version(projectname): - """Return a version which is equal to the tag that's on the current - revision if there is one, or tag plus number of additional revisions - if the current revision has no tag.""" - - if os.path.isdir('.git'): - version = _get_git_post_version() - write_versioninfo(projectname, version) - return version - return read_versioninfo(projectname) diff --git a/myjenkins.py b/myjenkins.py deleted file mode 100644 index cb16f1f7..00000000 --- a/myjenkins.py +++ /dev/null @@ -1,118 +0,0 @@ -import jenkins -import json -import urllib -import urllib2 -from jenkins import JenkinsException, NODE_TYPE, CREATE_NODE - -TOGGLE_OFFLINE = '/computer/%(name)s/toggleOffline?offlineMessage=%(msg)s' -CONFIG_NODE = '/computer/%(name)s/config.xml' - - -class Jenkins(jenkins.Jenkins): - def disable_node(self, name, msg=''): - ''' - Disable a node - - @param name: Jenkins node name - @type name: str - @param msg: Offline message - @type msg: str - ''' - info = self.get_node_info(name) - if info['offline']: - return - self.jenkins_open( - urllib2.Request(self.server + TOGGLE_OFFLINE % locals())) - - def enable_node(self, name): - ''' - Enable a node - - @param name: Jenkins node name - @type name: str - ''' - info = self.get_node_info(name) - if not info['offline']: - return - msg = '' - self.jenkins_open( - urllib2.Request(self.server + TOGGLE_OFFLINE % locals())) - - def get_node_config(self, name): - ''' - Get the configuration for a node. - - :param name: Jenkins node name, ``str`` - ''' - get_config_url = self.server + CONFIG_NODE % locals() - return self.jenkins_open(urllib2.Request(get_config_url)) - - def reconfig_node(self, name, config_xml): - ''' - Change the configuration for an existing node. - - :param name: Jenkins node name, ``str`` - :param config_xml: New XML configuration, ``str`` - ''' - headers = {'Content-Type': 'text/xml'} - reconfig_url = self.server + CONFIG_NODE % locals() - self.jenkins_open(urllib2.Request(reconfig_url, config_xml, headers)) - - def create_node(self, name, numExecutors=2, nodeDescription=None, - remoteFS='/var/lib/jenkins', labels=None, exclusive=False, - launcher='hudson.slaves.JNLPLauncher', launcher_params={}): - ''' - @param name: name of node to create - @type name: str - @param numExecutors: number of executors for node - @type numExecutors: int - @param nodeDescription: Description of node - @type nodeDescription: str - @param remoteFS: Remote filesystem location to use - @type remoteFS: str - @param labels: Labels to associate with node - @type labels: str - @param exclusive: Use this node for tied jobs only - @type exclusive: boolean - @param launcher: The launch method for the slave - @type launcher: str - @param launcher_params: Additional parameters for the launcher - @type launcher_params: dict - ''' - if self.node_exists(name): - raise JenkinsException('node[%s] already exists' % (name)) - - mode = 'NORMAL' - if exclusive: - mode = 'EXCLUSIVE' - - #hudson.plugins.sshslaves.SSHLauncher - #hudson.slaves.CommandLauncher - #hudson.os.windows.ManagedWindowsServiceLauncher - launcher_params['stapler-class'] = launcher - - inner_params = { - 'name': name, - 'nodeDescription': nodeDescription, - 'numExecutors': numExecutors, - 'remoteFS': remoteFS, - 'labelString': labels, - 'mode': mode, - 'type': NODE_TYPE, - 'retentionStrategy': { - 'stapler-class': 'hudson.slaves.RetentionStrategy$Always'}, - 'nodeProperties': {'stapler-class-bag': 'true'}, - 'launcher': launcher_params - } - - params = { - 'name': name, - 'type': NODE_TYPE, - 'json': json.dumps(inner_params) - } - - self.jenkins_open(urllib2.Request( - self.server + CREATE_NODE % urllib.urlencode(params))) - - if not self.node_exists(name): - raise JenkinsException('create[%s] failed' % (name)) diff --git a/openstack-common.conf b/openstack-common.conf deleted file mode 100644 index 02db14a6..00000000 --- a/openstack-common.conf +++ /dev/null @@ -1,7 +0,0 @@ -[DEFAULT] - -# The list of modules to copy from openstack-common -modules=setup - -# The base module to hold the copy of openstack.common -base=devstackgate diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index c929b566..00000000 --- a/setup.cfg +++ /dev/null @@ -1,10 +0,0 @@ -[egg_info] -tag_build = -tag_date = 0 -tag_svn_revision = 0 - -[nosetests] -verbosity=2 -detailed-errors=1 -cover-erase = true -cover-inclusive = true diff --git a/setup.py b/setup.py deleted file mode 100644 index 5fa6f4ac..00000000 --- a/setup.py +++ /dev/null @@ -1,27 +0,0 @@ -import os -import sys -import setuptools - -from devstackgate.openstack.common import setup as common_setup - -setuptools.setup( - name="devstack-gate", - version="2012.2", - description="Devstack gate scripts used by Openstack CI team", - url='https://github.com/openstack-infra/devstack-gate', - license='Apache', - author='Openstack CI team', - author_email='openstack@lists.launchpad.net', - packages=setuptools.find_packages(exclude=['tests', 'tests.*']), - cmdclass=common_setup.get_cmdclass(), - classifiers=[ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'Intended Audience :: Information Technology', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - ], - test_suite="nose.collector", -) diff --git a/sshclient.py b/sshclient.py deleted file mode 100644 index ed7a20ac..00000000 --- a/sshclient.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python - -# Update the base image that is used for devstack VMs. - -# Copyright (C) 2011-2012 OpenStack LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -import paramiko -import sys - - -class SSHClient(object): - def __init__(self, ip, username, password=None, pkey=None): - client = paramiko.SSHClient() - client.load_system_host_keys() - client.set_missing_host_key_policy(paramiko.WarningPolicy()) - client.connect(ip, username=username, password=password, pkey=pkey) - self.client = client - - def ssh(self, action, command): - stdin, stdout, stderr = self.client.exec_command(command) - print command - output = '' - for x in stdout: - output += x - sys.stdout.write(x) - ret = stdout.channel.recv_exit_status() - print stderr.read() - if ret: - raise Exception("Unable to %s" % action) - return output - - def scp(self, source, dest): - print 'copy', source, dest - ftp = self.client.open_sftp() - ftp.put(source, dest) - ftp.close() diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/test_vmdatabase.py b/tests/test_vmdatabase.py deleted file mode 100644 index e901b94a..00000000 --- a/tests/test_vmdatabase.py +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env python - -# Update the base image that is used for devstack VMs. - -# Copyright (C) 2012 OpenStack LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -import time - -import testtools -from testtools import content - -import vmdatabase - - -class TestVMDatabase(testtools.TestCase): - - def setUp(self): - super(TestVMDatabase, self).setUp() - self.db = vmdatabase.VMDatabase(':memory:') - - def test_add_provider(self): - provider = vmdatabase.Provider( - name='rackspace', driver='rackspace', - username='testuser', api_key='testapikey', - giftable=False) - self.db.session.add(provider) - self.db.commit() - provider = vmdatabase.Provider( - name='hpcloud', driver='openstack', - username='testuser', api_key='testapikey', - giftable=True) - self.db.session.add(provider) - self.db.commit() - - def test_add_base_image(self): - self.test_add_provider() - - provider = self.db.getProvider('rackspace') - provider.newBaseImage('oneiric', 1) - provider.newBaseImage('precise', 2) - - provider = self.db.getProvider('hpcloud') - provider.newBaseImage('oneiric', 1) - provider.newBaseImage('precise', 2) - - def test_add_snap_image(self): - self.test_add_base_image() - provider = self.db.getProvider('rackspace') - base_image1 = provider.getBaseImage('oneiric') - base_image2 = provider.getBaseImage('precise') - snapshot_image1 = base_image1.newSnapshotImage( - 'oneiric-1331683549', 1331683549, 201, 301) - snapshot_image2 = base_image2.newSnapshotImage( - 'precise-1331683549', 1331683549, 202, 301) - - hp_provider = self.db.getProvider('hpcloud') - hp_base_image1 = hp_provider.getBaseImage('oneiric') - hp_base_image2 = hp_provider.getBaseImage('precise') - hp_base_image1.newSnapshotImage( - 'oneiric-1331683549', 1331929410, 211, 311) - hp_base_image2.newSnapshotImage( - 'precise-1331683549', 1331929410, 212, 311) - - self.db.print_state() - assert(not base_image1.current_snapshot) - assert(not base_image2.current_snapshot) - - snapshot_image1.state = vmdatabase.READY - assert(base_image1.current_snapshot) - assert(not base_image2.current_snapshot) - assert(snapshot_image1 == base_image1.current_snapshot) - - snapshot_image2.state = vmdatabase.READY - assert(base_image1.current_snapshot) - assert(base_image2.current_snapshot) - assert(snapshot_image1 == base_image1.current_snapshot) - assert(snapshot_image2 == base_image2.current_snapshot) - - snapshot_image2_latest = base_image2.newSnapshotImage( - 'precise-1331683550', 1331683550, 203, 303) - assert(base_image1.current_snapshot) - assert(base_image2.current_snapshot) - assert(snapshot_image1 == base_image1.current_snapshot) - assert(snapshot_image2 == base_image2.current_snapshot) - - snapshot_image2_latest.state = vmdatabase.READY - assert(base_image1.current_snapshot) - assert(base_image2.current_snapshot) - assert(snapshot_image1 == base_image1.current_snapshot) - assert(snapshot_image2_latest == base_image2.current_snapshot) - - def test_add_machine(self): - self.test_add_snap_image() - provider = self.db.getProvider('rackspace') - base_image1 = provider.getBaseImage('oneiric') - base_image2 = provider.getBaseImage('precise') - base_image1.current_snapshot - base_image2.current_snapshot - assert(len(provider.machines) == 0) - assert(len(provider.ready_machines) == 0) - assert(len(provider.building_machines) == 0) - assert(len(base_image1.machines) == 0) - assert(len(base_image1.ready_machines) == 0) - assert(len(base_image1.building_machines) == 0) - assert(len(base_image2.machines) == 0) - assert(len(base_image2.ready_machines) == 0) - assert(len(base_image2.building_machines) == 0) - - machine1 = base_image1.newMachine('%s-1331683760' % base_image1.name, - '20000021', '1.2.3.4', 'uuid1') - assert(len(provider.machines) == 1) - assert(len(provider.ready_machines) == 0) - assert(len(provider.building_machines) == 1) - assert(len(base_image1.machines) == 1) - assert(len(base_image1.ready_machines) == 0) - assert(len(base_image1.building_machines) == 1) - assert(len(base_image2.machines) == 0) - assert(len(base_image2.ready_machines) == 0) - assert(len(base_image2.building_machines) == 0) - - machine2 = base_image2.newMachine('%s-1331683761' % base_image1.name, - '20000022', '1.2.3.5', 'uuid2') - assert(len(provider.machines) == 2) - assert(len(provider.ready_machines) == 0) - assert(len(provider.building_machines) == 2) - assert(len(base_image1.machines) == 1) - assert(len(base_image1.ready_machines) == 0) - assert(len(base_image1.building_machines) == 1) - assert(len(base_image2.machines) == 1) - assert(len(base_image2.ready_machines) == 0) - assert(len(base_image2.building_machines) == 1) - - machine1.state = vmdatabase.READY - assert(len(provider.machines) == 2) - assert(len(provider.ready_machines) == 1) - assert(len(provider.building_machines) == 1) - assert(len(base_image1.machines) == 1) - assert(len(base_image1.ready_machines) == 1) - assert(len(base_image1.building_machines) == 0) - assert(len(base_image2.machines) == 1) - assert(len(base_image2.ready_machines) == 0) - assert(len(base_image2.building_machines) == 1) - - machine2.state = vmdatabase.ERROR - assert(len(provider.machines) == 2) - assert(len(provider.ready_machines) == 1) - assert(len(provider.building_machines) == 0) - assert(len(base_image1.machines) == 1) - assert(len(base_image1.ready_machines) == 1) - assert(len(base_image1.building_machines) == 0) - assert(len(base_image2.machines) == 1) - assert(len(base_image2.ready_machines) == 0) - assert(len(base_image2.building_machines) == 0) - - machine2.state = vmdatabase.READY - assert(len(provider.machines) == 2) - assert(len(provider.ready_machines) == 2) - assert(len(provider.building_machines) == 0) - assert(len(base_image1.machines) == 1) - assert(len(base_image1.ready_machines) == 1) - assert(len(base_image1.building_machines) == 0) - assert(len(base_image2.machines) == 1) - assert(len(base_image2.ready_machines) == 1) - assert(len(base_image2.building_machines) == 0) - - hp_provider = self.db.getProvider('hpcloud') - hp_base_image1 = hp_provider.getBaseImage('oneiric') - hp_base_image2 = hp_provider.getBaseImage('precise') - hp_machine1 = hp_base_image1.newMachine( - '%s-1331683551' % hp_base_image1.name, - '21000021', '2.2.3.4', 'hpuuid1') - hp_machine2 = hp_base_image2.newMachine( - '%s-1331683552' % hp_base_image2.name, - '21000022', '2.2.3.5', 'hpuuid2') - hp_machine1.state = vmdatabase.READY - hp_machine2.state = vmdatabase.READY - - return (machine1, machine2, hp_machine1, hp_machine2) - - def test_get_machine(self): - (machine1, machine2, - hp_machine1, hp_machine2) = self.test_add_machine() - # order should be rs1, hp1 for oneiric, hp2, rs1 for precise - hp_machine2.state_time = machine1.state_time - 60 - self.db.commit() - - self.db.print_state() - - rs_provider = self.db.getProvider('rackspace') - hp_provider = self.db.getProvider('hpcloud') - - assert(len(rs_provider.ready_machines) == 2) - assert(len(hp_provider.ready_machines) == 2) - - machine = self.db.getMachineForUse('oneiric') - self.addDetail('machine name', content.text_content(machine.name)) - assert(len(rs_provider.ready_machines) == 1) - assert(len(hp_provider.ready_machines) == 2) - assert(machine == machine1) - - machine = self.db.getMachineForUse('oneiric') - self.addDetail('machine name', content.text_content(machine.name)) - assert(len(rs_provider.ready_machines) == 1) - assert(len(hp_provider.ready_machines) == 1) - assert(machine == hp_machine1) - - machine = self.db.getMachineForUse('precise') - self.addDetail('machine name', content.text_content(machine.name)) - assert(len(rs_provider.ready_machines) == 1) - assert(len(hp_provider.ready_machines) == 0) - assert(machine == hp_machine2) - - machine = self.db.getMachineForUse('precise') - self.addDetail('machine name', content.text_content(machine.name)) - assert(len(rs_provider.ready_machines) == 0) - assert(len(hp_provider.ready_machines) == 0) - assert(machine == machine2) - - def test_result(self): - (rs_machine1, rs_machine2, - hp_machine1, hp_machine2) = self.test_add_machine() - self.db.print_state() - - machine = self.db.getMachineForUse('oneiric') - self.addDetail('machine name', content.text_content(machine.name)) - result = machine.newResult('test-job', 82, 1234, 1) - time.sleep(2) - result.setResult(vmdatabase.RESULT_SUCCESS) - self.db.commit() - - orig_result = result - for provider in self.db.getProviders(): - for base_image in provider.base_images: - if (base_image.name == orig_result.base_image.name and - base_image.provider.name - == orig_result.base_image.provider.name): - assert(len(base_image.results) == 1) - for result in base_image.results: - assert(result.end_time > result.start_time) - assert(result.result == vmdatabase.RESULT_SUCCESS) - assert(result.machine_id == machine.id) - assert(result.jenkins_job_name == 'test-job') - assert(result.jenkins_build_number == 82) - assert(result.gerrit_change_number == 1234) - assert(result.gerrit_patchset_number == 1) - else: - assert(len(base_image.results) == 0) diff --git a/tools/pip-requires b/tools/pip-requires deleted file mode 100644 index 6c5b2691..00000000 --- a/tools/pip-requires +++ /dev/null @@ -1,4 +0,0 @@ -# devstack-gate dependencies -sqlalchemy -sqlalchemy-migrate -statsd diff --git a/tools/test-requires b/tools/test-requires deleted file mode 100644 index ce5788a9..00000000 --- a/tools/test-requires +++ /dev/null @@ -1,10 +0,0 @@ -# Testing - -# Install bounded pep8/pyflakes first, then let flake8 install -hacking>=0.5.3,<0.6 - -coverage>=3.6 -discover -python-subunit -testrepository>=0.0.13 -testtools>=0.9.27 diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 21da2d41..00000000 --- a/tox.ini +++ /dev/null @@ -1,21 +0,0 @@ -[tox] -envlist = py26,py27,pep8 - -[testenv] -setenv = VIRTUAL_ENV={envdir} -deps = -r{toxinidir}/tools/pip-requires - -r{toxinidir}/tools/test-requires -commands = - python setup.py testr --slowest --testr-args='{posargs}' - -[testenv:pep8] -commands = flake8 - -[testenv:cover] -commands = - python setup.py testr --coverage - -[flake8] -exclude = .git,.tox,dist,*egg,build,v1_0 -show-source = True -ignore = E12,H diff --git a/utils.py b/utils.py deleted file mode 100644 index 486b26ac..00000000 --- a/utils.py +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/env python - -# Update the base image that is used for devstack VMs. - -# Copyright (C) 2011-2012 OpenStack LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -import novaclient -from novaclient.v1_1 import client as Client11 -from v1_0 import client as Client10 -import time -import traceback -import paramiko -import socket -from sshclient import SSHClient -from statsd import statsd -import logging -import logging.handlers - -import vmdatabase - -log = logging.getLogger('devstack-gate') -log.setLevel(logging.DEBUG) -handler = logging.handlers.SysLogHandler(address='/dev/log') -handler.setFormatter(logging.Formatter("devstack-gate: %(message)s")) -log.addHandler(handler) - - -def iterate_timeout(max_seconds, purpose): - start = time.time() - count = 0 - while (time.time() < start + max_seconds): - count += 1 - yield count - time.sleep(2) - raise Exception("Timeout waiting for %s" % purpose) - - -def get_client(provider): - args = [provider.nova_username, provider.nova_api_key, - provider.nova_project_id, provider.nova_auth_url] - kwargs = {} - if provider.nova_service_type: - kwargs['service_type'] = provider.nova_service_type - if provider.nova_service_name: - kwargs['service_name'] = provider.nova_service_name - if provider.nova_service_region: - kwargs['region_name'] = provider.nova_service_region - if provider.nova_api_version == '1.0': - Client = Client10.Client - elif provider.nova_api_version == '1.1': - Client = Client11.Client - else: - raise Exception("API version not supported") - if provider.nova_rax_auth: - os.environ['NOVA_RAX_AUTH'] = '1' - client = Client(*args, **kwargs) - return client - -extension_cache = {} - - -def get_extensions(client): - global extension_cache - cache = extension_cache.get(client) - if cache: - return cache - try: - resp, body = client.client.get('/extensions') - extensions = [x['alias'] for x in body['extensions']] - except novaclient.exceptions.NotFound: - extensions = [] - extension_cache[client] = extensions - return extensions - - -def get_flavor(client, min_ram): - flavors = [f for f in client.flavors.list() if f.ram >= min_ram] - flavors.sort(lambda a, b: cmp(a.ram, b.ram)) - return flavors[0] - - -def get_public_ip(server, version=4): - if 'os-floating-ips' in get_extensions(server.manager.api): - print 'using floating ips' - for addr in server.manager.api.floating_ips.list(): - print 'checking addr', addr - if addr.instance_id == server.id: - print 'found addr', addr - return addr.ip - print 'no floating ip, addresses:' - print server.addresses - for addr in server.addresses.get('public', []): - if type(addr) == type(u''): # Rackspace/openstack 1.0 - return addr - if addr['version'] == version: # Rackspace/openstack 1.1 - return addr['addr'] - for addr in server.addresses.get('private', []): - # HPcloud - if addr['version'] == version and not addr['addr'].startswith('10.'): - return addr['addr'] - return None - - -def add_public_ip(server): - ip = server.manager.api.floating_ips.create() - print "created floating ip", ip - server.add_floating_ip(ip) - for count in iterate_timeout(600, "ip to be added"): - try: - newip = ip.manager.get(ip.id) - except: - print "Unable to get ip details, will retry" - traceback.print_exc() - continue - - if newip.instance_id == server.id: - print 'ip has been added' - return - - -def add_keypair(client, name): - key = paramiko.RSAKey.generate(2048) - public_key = key.get_name() + ' ' + key.get_base64() - kp = client.keypairs.create(name, public_key) - return key, kp - - -def wait_for_resource(wait_resource): - last_progress = None - last_status = None - # It can take a _very_ long time for Rackspace 1.0 to save an image - for count in iterate_timeout(21600, "waiting for %s" % wait_resource): - try: - resource = wait_resource.manager.get(wait_resource.id) - except: - print "Unable to list resources, will retry" - traceback.print_exc() - continue - - # In Rackspace v1.0, there is no progress attribute while queued - if hasattr(resource, 'progress'): - if (last_progress != resource.progress - or last_status != resource.status): - print resource.status, resource.progress - last_progress = resource.progress - elif last_status != resource.status: - print resource.status - last_status = resource.status - if resource.status == 'ACTIVE': - return resource - - -def ssh_connect(ip, username, connect_kwargs={}, timeout=60): - # HPcloud may return errno 111 for about 30 seconds after adding the IP - for count in iterate_timeout(timeout, "ssh access"): - try: - client = SSHClient(ip, username, **connect_kwargs) - break - except socket.error, e: - print "While testing ssh access:", e - - out = client.ssh("test ssh access", "echo access okay") - if "access okay" in out: - return client - return None - - -def delete_server(server): - try: - if 'os-floating-ips' in get_extensions(server.manager.api): - for addr in server.manager.api.floating_ips.list(): - if addr.instance_id == server.id: - server.remove_floating_ip(addr) - addr.delete() - except: - print "Unable to remove floating IP" - traceback.print_exc() - - try: - if 'os-keypairs' in get_extensions(server.manager.api): - for kp in server.manager.api.keypairs.list(): - if kp.name == server.key_name: - kp.delete() - except: - print "Unable to delete keypair" - traceback.print_exc() - - print "Deleting server", server.id - server.delete() - - -def update_stats(provider): - state_names = { - vmdatabase.BUILDING: 'building', - vmdatabase.READY: 'ready', - vmdatabase.USED: 'used', - vmdatabase.ERROR: 'error', - vmdatabase.HOLD: 'hold', - vmdatabase.DELETE: 'delete', - } - - for base_image in provider.base_images: - states = { - vmdatabase.BUILDING: 0, - vmdatabase.READY: 0, - vmdatabase.USED: 0, - vmdatabase.ERROR: 0, - vmdatabase.HOLD: 0, - vmdatabase.DELETE: 0, - } - for machine in base_image.machines: - if machine.state not in states: - continue - states[machine.state] += 1 - if statsd: - for state_id, count in states.items(): - key = 'devstack.pool.%s.%s.%s' % ( - provider.name, - base_image.name, - state_names[state_id]) - statsd.gauge(key, count) - - key = 'devstack.pool.%s.%s.min_ready' % ( - provider.name, - base_image.name) - statsd.gauge(key, base_image.min_ready) - - if statsd: - key = 'devstack.pool.%s.max_servers' % provider.name - statsd.gauge(key, provider.max_servers) diff --git a/v1_0/__init__.py b/v1_0/__init__.py deleted file mode 100644 index 32c1be5f..00000000 --- a/v1_0/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from client import Client diff --git a/v1_0/accounts.py b/v1_0/accounts.py deleted file mode 100644 index 4a455852..00000000 --- a/v1_0/accounts.py +++ /dev/null @@ -1,18 +0,0 @@ -from novaclient import base -import base as local_base - - -class Account(base.Resource): - pass - - -class AccountManager(local_base.BootingManagerWithFind): - resource_class = Account - - def create_instance_for(self, account_id, name, image, flavor, - ipgroup=None, meta=None, files=None, zone_blob=None, - reservation_id=None): - resource_url = "/accounts/%s/create_instance" % account_id - return self._boot(resource_url, "server", name, image, flavor, - ipgroup=ipgroup, meta=meta, files=files, - zone_blob=zone_blob, reservation_id=reservation_id) diff --git a/v1_0/backup_schedules.py b/v1_0/backup_schedules.py deleted file mode 100644 index 2d8aea82..00000000 --- a/v1_0/backup_schedules.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2010 Jacob Kaplan-Moss -""" -Backup Schedule interface. -""" - -from novaclient import base - - -BACKUP_WEEKLY_DISABLED = 'DISABLED' -BACKUP_WEEKLY_SUNDAY = 'SUNDAY' -BACKUP_WEEKLY_MONDAY = 'MONDAY' -BACKUP_WEEKLY_TUESDAY = 'TUESDAY' -BACKUP_WEEKLY_WEDNESDAY = 'WEDNESDAY' -BACKUP_WEEKLY_THURSDAY = 'THURSDAY' -BACKUP_WEEKLY_FRIDAY = 'FRIDAY' -BACKUP_WEEKLY_SATURDAY = 'SATURDAY' - -BACKUP_DAILY_DISABLED = 'DISABLED' -BACKUP_DAILY_H_0000_0200 = 'H_0000_0200' -BACKUP_DAILY_H_0200_0400 = 'H_0200_0400' -BACKUP_DAILY_H_0400_0600 = 'H_0400_0600' -BACKUP_DAILY_H_0600_0800 = 'H_0600_0800' -BACKUP_DAILY_H_0800_1000 = 'H_0800_1000' -BACKUP_DAILY_H_1000_1200 = 'H_1000_1200' -BACKUP_DAILY_H_1200_1400 = 'H_1200_1400' -BACKUP_DAILY_H_1400_1600 = 'H_1400_1600' -BACKUP_DAILY_H_1600_1800 = 'H_1600_1800' -BACKUP_DAILY_H_1800_2000 = 'H_1800_2000' -BACKUP_DAILY_H_2000_2200 = 'H_2000_2200' -BACKUP_DAILY_H_2200_0000 = 'H_2200_0000' - - -class BackupSchedule(base.Resource): - """ - Represents the daily or weekly backup schedule for some server. - """ - def get(self): - """ - Get this `BackupSchedule` again from the API. - """ - return self.manager.get(server=self.server) - - def delete(self): - """ - Delete (i.e. disable and remove) this scheduled backup. - """ - self.manager.delete(server=self.server) - - def update(self, enabled=True, weekly=BACKUP_WEEKLY_DISABLED, - daily=BACKUP_DAILY_DISABLED): - """ - Update this backup schedule. - - See :meth:`BackupScheduleManager.create` for details. - """ - self.manager.create(self.server, enabled, weekly, daily) - - -class BackupScheduleManager(base.Manager): - """ - Manage server backup schedules. - """ - resource_class = BackupSchedule - - def get(self, server): - """ - Get the current backup schedule for a server. - - :arg server: The server (or its ID). - :rtype: :class:`BackupSchedule` - """ - s = base.getid(server) - schedule = self._get('/servers/%s/backup_schedule' % s, - 'backupSchedule') - schedule.server = server - return schedule - - # Backup schedules use POST for both create and update, so allow both here. - # Unlike the rest of the API, POST here returns no body, so we can't use - # the nice little helper methods. - - def create(self, server, enabled=True, weekly=BACKUP_WEEKLY_DISABLED, - daily=BACKUP_DAILY_DISABLED): - """ - Create or update the backup schedule for the given server. - - :arg server: The server (or its ID). - :arg enabled: boolean; should this schedule be enabled? - :arg weekly: Run a weekly backup on this day - (one of the `BACKUP_WEEKLY_*` constants) - :arg daily: Run a daily backup at this time - (one of the `BACKUP_DAILY_*` constants) - """ - s = base.getid(server) - body = {'backupSchedule': { - 'enabled': enabled, 'weekly': weekly, 'daily': daily - }} - self.api.client.post('/servers/%s/backup_schedule' % s, body=body) - - update = create - - def delete(self, server): - """ - Remove the scheduled backup for `server`. - - :arg server: The server (or its ID). - """ - s = base.getid(server) - self._delete('/servers/%s/backup_schedule' % s) diff --git a/v1_0/base.py b/v1_0/base.py deleted file mode 100644 index f03eb615..00000000 --- a/v1_0/base.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2010 Jacob Kaplan-Moss - -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Base utilities to build API operation managers and objects on top of. -""" - -from novaclient import base - - -# Python 2.4 compat -try: - all -except NameError: - def all(iterable): - return True not in (not x for x in iterable) - - -class BootingManagerWithFind(base.ManagerWithFind): - """Like a `ManagerWithFind`, but has the ability to boot servers.""" - def _boot(self, resource_url, response_key, name, image, flavor, - ipgroup=None, meta=None, files=None, zone_blob=None, - reservation_id=None, return_raw=False, min_count=None, - max_count=None): - """ - Create (boot) a new server. - - :param name: Something to name the server. - :param image: The :class:`Image` to boot with. - :param flavor: The :class:`Flavor` to boot onto. - :param ipgroup: An initial :class:`IPGroup` for this server. - :param meta: A dict of arbitrary key/value metadata to store for this - server. A maximum of five entries is allowed, and both - keys and values must be 255 characters or less. - :param files: A dict of files to overrwrite on the server upon boot. - Keys are file names (i.e. ``/etc/passwd``) and values - are the file contents (either as a string or as a - file-like object). A maximum of five entries is allowed, - and each file must be 10k or less. - :param zone_blob: a single (encrypted) string which is used internally - by Nova for routing between Zones. Users cannot populate - this field. - :param reservation_id: a UUID for the set of servers being requested. - :param return_raw: If True, don't try to coearse the result into - a Resource object. - """ - body = {"server": { - "name": name, - "imageId": base.getid(image), - "flavorId": base.getid(flavor), - }} - if ipgroup: - body["server"]["sharedIpGroupId"] = base.getid(ipgroup) - if meta: - body["server"]["metadata"] = meta - if reservation_id: - body["server"]["reservation_id"] = reservation_id - if zone_blob: - body["server"]["blob"] = zone_blob - - if not min_count: - min_count = 1 - if not max_count: - max_count = min_count - body["server"]["min_count"] = min_count - body["server"]["max_count"] = max_count - - # Files are a slight bit tricky. They're passed in a "personality" - # list to the POST. Each item is a dict giving a file name and the - # base64-encoded contents of the file. We want to allow passing - # either an open file *or* some contents as files here. - if files: - personality = body['server']['personality'] = [] - for filepath, file_or_string in files.items(): - if hasattr(file_or_string, 'read'): - data = file_or_string.read() - else: - data = file_or_string - personality.append({ - 'path': filepath, - 'contents': data.encode('base64'), - }) - - return self._create(resource_url, body, response_key, - return_raw=return_raw) diff --git a/v1_0/client.py b/v1_0/client.py deleted file mode 100644 index ae07c5ef..00000000 --- a/v1_0/client.py +++ /dev/null @@ -1,75 +0,0 @@ -from novaclient import client -import accounts -import backup_schedules -import flavors -import images -import ipgroups -import servers -import zones - - -class Client(object): - """ - Top-level object to access the OpenStack Compute API. - - Create an instance with your creds:: - - >>> client = Client(USERNAME, PASSWORD, PROJECT_ID, AUTH_URL) - - Then call methods on its managers:: - - >>> client.servers.list() - ... - >>> client.flavors.list() - ... - - """ - - def __init__(self, username, api_key, project_id, auth_url=None, - insecure=False, timeout=None, token=None, region_name=None, - endpoint_name=None, extensions=None, service_type=None, - service_name=None, endpoint_type='publicURL'): - - # FIXME(comstud): Rename the api_key argument above when we - # know it's not being used as keyword argument - password = api_key - self.accounts = accounts.AccountManager(self) - self.backup_schedules = backup_schedules.BackupScheduleManager(self) - self.flavors = flavors.FlavorManager(self) - self.images = images.ImageManager(self) - self.ipgroups = ipgroups.IPGroupManager(self) - self.servers = servers.ServerManager(self) - self.zones = zones.ZoneManager(self) - #service_type is unused in v1_0 - #service_name is unused in v1_0 - #endpoint_name is unused in v_10 - #endpoint_type was endpoint_name - - # Add in any extensions... - if extensions: - for (ext_name, ext_manager_class, ext_module) in extensions: - setattr(self, ext_name, ext_manager_class(self)) - - _auth_url = auth_url or 'https://auth.api.rackspacecloud.com/v1.0' - - self.client = client.HTTPClient(username, - password, - project_id, - _auth_url, - insecure=insecure, - timeout=timeout, - proxy_token=token, - region_name=region_name, - endpoint_type=endpoint_type) - - def authenticate(self): - """ - Authenticate against the server. - - Normally this is called automatically when you first access the API, - but you can call this method to force authentication right now. - - Returns on success; raises :exc:`exceptions.Unauthorized` if the - credentials are wrong. - """ - self.client.authenticate() diff --git a/v1_0/flavors.py b/v1_0/flavors.py deleted file mode 100644 index f1b49580..00000000 --- a/v1_0/flavors.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2010 Jacob Kaplan-Moss -""" -Flavor interface. -""" - -from novaclient import base - - -class Flavor(base.Resource): - """ - A flavor is an available hardware configuration for a server. - """ - def __repr__(self): - return "" % self.name - - -class FlavorManager(base.ManagerWithFind): - """ - Manage :class:`Flavor` resources. - """ - resource_class = Flavor - - def list(self, detailed=True): - """ - Get a list of all flavors. - - :rtype: list of :class:`Flavor`. - """ - detail = "" - if detailed: - detail = "/detail" - return self._list("/flavors%s" % detail, "flavors") - - def get(self, flavor): - """ - Get a specific flavor. - - :param flavor: The ID of the :class:`Flavor` to get. - :rtype: :class:`Flavor` - """ - return self._get("/flavors/%s" % base.getid(flavor), "flavor") diff --git a/v1_0/images.py b/v1_0/images.py deleted file mode 100644 index cf063cfe..00000000 --- a/v1_0/images.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2010 Jacob Kaplan-Moss -""" -Image interface. -""" - -from novaclient import base -import time - -class Image(base.Resource): - """ - An image is a collection of files used to create or rebuild a server. - """ - def __repr__(self): - return "" % self.name - - def delete(self): - """ - Delete this image. - """ - return self.manager.delete(self) - - -class ImageManager(base.ManagerWithFind): - """ - Manage :class:`Image` resources. - """ - resource_class = Image - - def get(self, image): - """ - Get an image. - - :param image: The ID of the image to get. - :rtype: :class:`Image` - """ - return self._get("/images/%s" % base.getid(image), "image") - - def list(self, detailed=True): - """ - Get a list of all images. - - :rtype: list of :class:`Image` - """ - detail = "" - if detailed: - detail = "/detail" - return self._list("/images%s?cache-busting=%s" % (detail, time.time()), "images") - - def create(self, server, name): - """ - Create a new image by snapshotting a running :class:`Server` - - :param name: An (arbitrary) name for the new image. - :param server: The :class:`Server` (or its ID) to make a snapshot of. - :rtype: :class:`Image` - """ - data = {"image": {"serverId": base.getid(server), "name": name}} - return self._create("/images", data, "image") - - def delete(self, image): - """ - Delete an image. - - It should go without saying that you can't delete an image - that you didn't create. - - :param image: The :class:`Image` (or its ID) to delete. - """ - self._delete("/images/%s" % base.getid(image)) diff --git a/v1_0/ipgroups.py b/v1_0/ipgroups.py deleted file mode 100644 index 86cd3cb4..00000000 --- a/v1_0/ipgroups.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2010 Jacob Kaplan-Moss -""" -IP Group interface. -""" - -from novaclient import base - - -class IPGroup(base.Resource): - def __repr__(self): - return "" % self.name - - def delete(self): - """ - Delete this group. - """ - self.manager.delete(self) - - -class IPGroupManager(base.ManagerWithFind): - resource_class = IPGroup - - def list(self, detailed=True): - """ - Get a list of all groups. - - :rtype: list of :class:`IPGroup` - """ - detail = "" - if detailed: - detail = "/detail" - return self._list("/shared_ip_groups%s" % detail, "sharedIpGroups") - - def get(self, group): - """ - Get an IP group. - - :param group: ID of the image to get. - :rtype: :class:`IPGroup` - """ - return self._get("/shared_ip_groups/%s" % base.getid(group), - "sharedIpGroup") - - def create(self, name, server=None): - """ - Create a new :class:`IPGroup` - - :param name: An (arbitrary) name for the new image. - :param server: A :class:`Server` (or its ID) to make a member - of this group. - :rtype: :class:`IPGroup` - """ - data = {"sharedIpGroup": {"name": name}} - if server: - data['sharedIpGroup']['server'] = base.getid(server) - return self._create('/shared_ip_groups', data, "sharedIpGroup") - - def delete(self, group): - """ - Delete a group. - - :param group: The :class:`IPGroup` (or its ID) to delete. - """ - self._delete("/shared_ip_groups/%s" % base.getid(group)) diff --git a/v1_0/servers.py b/v1_0/servers.py deleted file mode 100644 index d4e8f393..00000000 --- a/v1_0/servers.py +++ /dev/null @@ -1,488 +0,0 @@ -# Copyright 2010 Jacob Kaplan-Moss - -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Server interface. -""" - -import urllib - -from novaclient import base -import base as local_base -import time - -REBOOT_SOFT, REBOOT_HARD = 'SOFT', 'HARD' - - -class Server(base.Resource): - def __repr__(self): - return "" % self.name - - def delete(self): - """ - Delete (i.e. shut down and delete the image) this server. - """ - self.manager.delete(self) - - def update(self, name=None, password=None): - """ - Update the name or the password for this server. - - :param name: Update the server's name. - :param password: Update the root password. - """ - self.manager.update(self, name, password) - - def share_ip(self, ipgroup, address, configure=True): - """ - Share an IP address from the given IP group onto this server. - - :param ipgroup: The :class:`IPGroup` that the given address belongs to. - :param address: The IP address to share. - :param configure: If ``True``, the server will be automatically - configured to use this IP. I don't know why you'd - want this to be ``False``. - """ - self.manager.share_ip(self, ipgroup, address, configure) - - def unshare_ip(self, address): - """ - Stop sharing the given address. - - :param address: The IP address to stop sharing. - """ - self.manager.unshare_ip(self, address) - - def add_fixed_ip(self, network_id): - """ - Add an IP address on a network. - - :param network_id: The ID of the network the IP should be on. - """ - self.manager.add_fixed_ip(self, network_id) - - def remove_fixed_ip(self, address): - """ - Remove an IP address. - - :param address: The IP address to remove. - """ - self.manager.remove_fixed_ip(self, address) - - def reboot(self, type=REBOOT_SOFT): - """ - Reboot the server. - - :param type: either :data:`REBOOT_SOFT` for a software-level reboot, - or `REBOOT_HARD` for a virtual power cycle hard reboot. - """ - self.manager.reboot(self, type) - - def pause(self): - """ - Pause -- Pause the running server. - """ - self.manager.pause(self) - - def unpause(self): - """ - Unpause -- Unpause the paused server. - """ - self.manager.unpause(self) - - def suspend(self): - """ - Suspend -- Suspend the running server. - """ - self.manager.suspend(self) - - def resume(self): - """ - Resume -- Resume the suspended server. - """ - self.manager.resume(self) - - def rescue(self): - """ - Rescue -- Rescue the problematic server. - """ - self.manager.rescue(self) - - def unrescue(self): - """ - Unrescue -- Unrescue the rescued server. - """ - self.manager.unrescue(self) - - def diagnostics(self): - """Diagnostics -- Retrieve server diagnostics.""" - self.manager.diagnostics(self) - - def actions(self): - """Actions -- Retrieve server actions.""" - self.manager.actions(self) - - def rebuild(self, image): - """ - Rebuild -- shut down and then re-image -- this server. - - :param image: the :class:`Image` (or its ID) to re-image with. - """ - self.manager.rebuild(self, image) - - def resize(self, flavor): - """ - Resize the server's resources. - - :param flavor: the :class:`Flavor` (or its ID) to resize to. - - Until a resize event is confirmed with :meth:`confirm_resize`, the old - server will be kept around and you'll be able to roll back to the old - flavor quickly with :meth:`revert_resize`. All resizes are - automatically confirmed after 24 hours. - """ - self.manager.resize(self, flavor) - - def backup(self, image_name, backup_type, rotation): - """ - Create a server backup. - - :param server: The :class:`Server` (or its ID). - :param image_name: The name to assign the newly create image. - :param backup_type: 'daily' or 'weekly' - :param rotation: number of backups of type 'backup_type' to keep - :returns Newly created :class:`Image` object - """ - return self.manager.backup(self, image_name, backup_type, rotation) - - def confirm_resize(self): - """ - Confirm that the resize worked, thus removing the original server. - """ - self.manager.confirm_resize(self) - - def revert_resize(self): - """ - Revert a previous resize, switching back to the old server. - """ - self.manager.revert_resize(self) - - def migrate(self): - """ - Migrate a server to a new host in the same zone. - """ - self.manager.migrate(self) - - @property - def backup_schedule(self): - """ - This server's :class:`BackupSchedule`. - """ - return self.manager.api.backup_schedules.get(self) - - @property - def public_ip(self): - """ - Shortcut to get this server's primary public IP address. - """ - if len(self.addresses['public']) == 0: - return "" - return self.addresses['public'] - - @property - def private_ip(self): - """ - Shortcut to get this server's primary private IP address. - """ - if len(self.addresses['private']) == 0: - return "" - return self.addresses['private'] - - -class ServerManager(local_base.BootingManagerWithFind): - resource_class = Server - - def get(self, server): - """ - Get a server. - - :param server: ID of the :class:`Server` to get. - :rtype: :class:`Server` - """ - return self._get("/servers/%s" % base.getid(server), "server") - - def list(self, detailed=True, search_opts=None): - """ - Get a list of servers. - Optional detailed returns details server info. - Optional reservation_id only returns instances with that - reservation_id. - - :rtype: list of :class:`Server` - """ - if search_opts is None: - search_opts = {} - qparams = {} - # only use values in query string if they are set - for opt, val in search_opts.iteritems(): - if val: - qparams[opt] = val - - query_string = "?%s" % urllib.urlencode(qparams) if qparams else "" - - detail = "" - if detailed: - detail = "/detail" - return self._list("/servers%s%s?cache-busting=%s" % (detail, query_string, time.time()), "servers") - - def create(self, name, image, flavor, ipgroup=None, meta=None, files=None, - zone_blob=None, reservation_id=None, min_count=None, - max_count=None): - """ - Create (boot) a new server. - - :param name: Something to name the server. - :param image: The :class:`Image` to boot with. - :param flavor: The :class:`Flavor` to boot onto. - :param ipgroup: An initial :class:`IPGroup` for this server. - :param meta: A dict of arbitrary key/value metadata to store for this - server. A maximum of five entries is allowed, and both - keys and values must be 255 characters or less. - :param files: A dict of files to overrwrite on the server upon boot. - Keys are file names (i.e. ``/etc/passwd``) and values - are the file contents (either as a string or as a - file-like object). A maximum of five entries is allowed, - and each file must be 10k or less. - :param zone_blob: a single (encrypted) string which is used internally - by Nova for routing between Zones. Users cannot populate - this field. - :param reservation_id: a UUID for the set of servers being requested. - """ - if not min_count: - min_count = 1 - if not max_count: - max_count = min_count - if min_count > max_count: - min_count = max_count - return self._boot("/servers", "server", name, image, flavor, - ipgroup=ipgroup, meta=meta, files=files, - zone_blob=zone_blob, reservation_id=reservation_id, - min_count=min_count, max_count=max_count) - - def update(self, server, name=None, password=None): - """ - Update the name or the password for a server. - - :param server: The :class:`Server` (or its ID) to update. - :param name: Update the server's name. - :param password: Update the root password. - """ - - if name is None and password is None: - return - body = {"server": {}} - if name: - body["server"]["name"] = name - if password: - body["server"]["adminPass"] = password - self._update("/servers/%s" % base.getid(server), body) - - def delete(self, server): - """ - Delete (i.e. shut down and delete the image) this server. - """ - self._delete("/servers/%s" % base.getid(server)) - - def share_ip(self, server, ipgroup, address, configure=True): - """ - Share an IP address from the given IP group onto a server. - - :param server: The :class:`Server` (or its ID) to share onto. - :param ipgroup: The :class:`IPGroup` that the given address belongs to. - :param address: The IP address to share. - :param configure: If ``True``, the server will be automatically - configured to use this IP. I don't know why you'd - want this to be ``False``. - """ - server = base.getid(server) - ipgroup = base.getid(ipgroup) - body = {'shareIp': {'sharedIpGroupId': ipgroup, - 'configureServer': configure}} - self._update("/servers/%s/ips/public/%s" % (server, address), body) - - def unshare_ip(self, server, address): - """ - Stop sharing the given address. - - :param server: The :class:`Server` (or its ID) to share onto. - :param address: The IP address to stop sharing. - """ - server = base.getid(server) - self._delete("/servers/%s/ips/public/%s" % (server, address)) - - def add_fixed_ip(self, server, network_id): - """ - Add an IP address on a network. - - :param server: The :class:`Server` (or its ID) to add an IP to. - :param network_id: The ID of the network the IP should be on. - """ - self._action('addFixedIp', server, {'networkId': network_id}) - - def remove_fixed_ip(self, server, address): - """ - Remove an IP address. - - :param server: The :class:`Server` (or its ID) to add an IP to. - :param address: The IP address to remove. - """ - self._action('removeFixedIp', server, {'address': address}) - - def reboot(self, server, type=REBOOT_SOFT): - """ - Reboot a server. - - :param server: The :class:`Server` (or its ID) to share onto. - :param type: either :data:`REBOOT_SOFT` for a software-level reboot, - or `REBOOT_HARD` for a virtual power cycle hard reboot. - """ - self._action('reboot', server, {'type': type}) - - def rebuild(self, server, image): - """ - Rebuild -- shut down and then re-image -- a server. - - :param server: The :class:`Server` (or its ID) to share onto. - :param image: the :class:`Image` (or its ID) to re-image with. - """ - self._action('rebuild', server, {'imageId': base.getid(image)}) - - def resize(self, server, flavor): - """ - Resize a server's resources. - - :param server: The :class:`Server` (or its ID) to share onto. - :param flavor: the :class:`Flavor` (or its ID) to resize to. - - Until a resize event is confirmed with :meth:`confirm_resize`, the old - server will be kept around and you'll be able to roll back to the old - flavor quickly with :meth:`revert_resize`. All resizes are - automatically confirmed after 24 hours. - """ - self._action('resize', server, {'flavorId': base.getid(flavor)}) - - def backup(self, server, image_name, backup_type, rotation): - """ - Create a server backup. - - :param server: The :class:`Server` (or its ID). - :param image_name: The name to assign the newly create image. - :param backup_type: 'daily' or 'weekly' - :param rotation: number of backups of type 'backup_type' to keep - :returns Newly created :class:`Image` object - """ - if not rotation: - raise Exception("rotation is required for backups") - elif not backup_type: - raise Exception("backup_type required for backups") - elif backup_type not in ("daily", "weekly"): - raise Exception("Invalid backup_type: must be daily or weekly") - - data = { - "name": image_name, - "rotation": rotation, - "backup_type": backup_type, - } - - self._action('createBackup', server, data) - - def pause(self, server): - """ - Pause the server. - """ - self.api.client.post('/servers/%s/pause' % base.getid(server)) - - def unpause(self, server): - """ - Unpause the server. - """ - self.api.client.post('/servers/%s/unpause' % base.getid(server)) - - def suspend(self, server): - """ - Suspend the server. - """ - self.api.client.post('/servers/%s/suspend' % base.getid(server)) - - def resume(self, server): - """ - Resume the server. - """ - self.api.client.post('/servers/%s/resume' % base.getid(server)) - - def rescue(self, server): - """ - Rescue the server. - """ - self.api.client.post('/servers/%s/rescue' % base.getid(server)) - - def unrescue(self, server): - """ - Unrescue the server. - """ - self.api.client.post('/servers/%s/unrescue' % base.getid(server)) - - def diagnostics(self, server): - """Retrieve server diagnostics.""" - return self.api.client.get("/servers/%s/diagnostics" % - base.getid(server)) - - def actions(self, server): - """Retrieve server actions.""" - return self._list("/servers/%s/actions" % base.getid(server), - "actions") - - def confirm_resize(self, server): - """ - Confirm that the resize worked, thus removing the original server. - - :param server: The :class:`Server` (or its ID) to share onto. - """ - self._action('confirmResize', server) - - def revert_resize(self, server): - """ - Revert a previous resize, switching back to the old server. - - :param server: The :class:`Server` (or its ID) to share onto. - """ - self._action('revertResize', server) - - def migrate(self, server): - """ - Migrate a server to a new host in the same zone. - - :param server: The :class:`Server` (or its ID). - """ - self.api.client.post('/servers/%s/migrate' % base.getid(server)) - - def _action(self, action, server, info=None): - """ - Perform a server "action" -- reboot/rebuild/resize/etc. - """ - self.api.client.post('/servers/%s/action' % base.getid(server), - body={action: info}) diff --git a/v1_0/shell.py b/v1_0/shell.py deleted file mode 100644 index cb462727..00000000 --- a/v1_0/shell.py +++ /dev/null @@ -1,788 +0,0 @@ -# Copyright 2010 Jacob Kaplan-Moss - -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import getpass -import os - -from novaclient import exceptions -from novaclient import utils -import client -import backup_schedules -import servers - - -CLIENT_CLASS = client.Client - -# Choices for flags. -DAY_CHOICES = [getattr(backup_schedules, i).lower() - for i in dir(backup_schedules) - if i.startswith('BACKUP_WEEKLY_')] -HOUR_CHOICES = [getattr(backup_schedules, i).lower() - for i in dir(backup_schedules) - if i.startswith('BACKUP_DAILY_')] - - -# Sentinal for boot --key -AUTO_KEY = object() - - -@utils.arg('server', metavar='', help='Name or ID of server.') -@utils.arg('--enable', dest='enabled', default=None, action='store_true', - help='Enable backups.') -@utils.arg('--disable', dest='enabled', action='store_false', - help='Disable backups.') -@utils.arg('--weekly', metavar='', choices=DAY_CHOICES, - help='Schedule a weekly backup for (one of: %s).' % - utils.pretty_choice_list(DAY_CHOICES)) -@utils.arg('--daily', metavar='', choices=HOUR_CHOICES, - help='Schedule a daily backup during (one of: %s).' % - utils.pretty_choice_list(HOUR_CHOICES)) -def do_backup_schedule(cs, args): - """ - Show or edit the backup schedule for a server. - - With no flags, the backup schedule will be shown. If flags are given, - the backup schedule will be modified accordingly. - """ - server = _find_server(cs, args.server) - - # If we have some flags, update the backup - backup = {} - if args.daily: - backup['daily'] = getattr(backup_schedules, 'BACKUP_DAILY_%s' % - args.daily.upper()) - if args.weekly: - backup['weekly'] = getattr(backup_schedules, 'BACKUP_WEEKLY_%s' % - args.weekly.upper()) - if args.enabled is not None: - backup['enabled'] = args.enabled - if backup: - server.backup_schedule.update(**backup) - else: - utils.print_dict(server.backup_schedule._info) - - -@utils.arg('server', metavar='', help='Name or ID of server.') -def do_backup_schedule_delete(cs, args): - """ - Delete the backup schedule for a server. - """ - server = _find_server(cs, args.server) - server.backup_schedule.delete() - - -def _boot(cs, args, reservation_id=None, min_count=None, max_count=None): - """Boot a new server.""" - if min_count is None: - min_count = 1 - if max_count is None: - max_count = min_count - if min_count > max_count: - raise exceptions.CommandError("min_instances should be" - "<= max_instances") - if not min_count or not max_count: - raise exceptions.CommandError("min_instances nor max_instances" - "should be 0") - - flavor = args.flavor or cs.flavors.find(ram=256) - image = args.image or cs.images.find(name="Ubuntu 10.04 LTS "\ - "(lucid)") - - # Map --ipgroup to an ID. - # XXX do this for flavor/image? - if args.ipgroup: - ipgroup = _find_ipgroup(cs, args.ipgroup) - else: - ipgroup = None - - metadata = dict(v.split('=') for v in args.meta) - - files = {} - for f in args.files: - dst, src = f.split('=', 1) - try: - files[dst] = open(src) - except IOError, e: - raise exceptions.CommandError("Can't open '%s': %s" % (src, e)) - - if args.key is AUTO_KEY: - possible_keys = [os.path.join(os.path.expanduser('~'), '.ssh', k) - for k in ('id_dsa.pub', 'id_rsa.pub')] - for k in possible_keys: - if os.path.exists(k): - keyfile = k - break - else: - raise exceptions.CommandError("Couldn't find a key file: tried " - "~/.ssh/id_dsa.pub or ~/.ssh/id_rsa.pub") - elif args.key: - keyfile = args.key - else: - keyfile = None - - if keyfile: - try: - files['/root/.ssh/authorized_keys2'] = open(keyfile) - except IOError, e: - raise exceptions.CommandError("Can't open '%s': %s" % (keyfile, e)) - - return (args.name, image, flavor, ipgroup, metadata, files, - reservation_id, min_count, max_count) - - -@utils.arg('--flavor', - default=None, - type=int, - metavar='', - help="Flavor ID (see 'nova flavors'). "\ - "Defaults to 256MB RAM instance.") -@utils.arg('--image', - default=None, - type=int, - metavar='', - help="Image ID (see 'nova images'). "\ - "Defaults to Ubuntu 10.04 LTS.") -@utils.arg('--ipgroup', - default=None, - metavar='', - help="IP group name or ID (see 'nova ipgroup-list').") -@utils.arg('--meta', - metavar="", - action='append', - default=[], - help="Record arbitrary key/value metadata. "\ - "May be give multiple times.") -@utils.arg('--file', - metavar="", - action='append', - dest='files', - default=[], - help="Store arbitrary files from locally to "\ - "on the new server. You may store up to 5 files.") -@utils.arg('--key', - metavar='', - nargs='?', - const=AUTO_KEY, - help="Key the server with an SSH keypair. "\ - "Looks in ~/.ssh for a key, "\ - "or takes an explicit to one.") -@utils.arg('name', metavar='', help='Name for the new server') -def do_boot(cs, args): - """Boot a new server.""" - name, image, flavor, ipgroup, metadata, files, reservation_id, \ - min_count, max_count = _boot(cs, args) - - server = cs.servers.create(args.name, image, flavor, - ipgroup=ipgroup, - meta=metadata, - files=files, - min_count=min_count, - max_count=max_count) - utils.print_dict(server._info) - - -@utils.arg('--flavor', - default=None, - type=int, - metavar='', - help="Flavor ID (see 'nova flavors'). "\ - "Defaults to 256MB RAM instance.") -@utils.arg('--image', - default=None, - type=int, - metavar='', - help="Image ID (see 'nova images'). "\ - "Defaults to Ubuntu 10.04 LTS.") -@utils.arg('--ipgroup', - default=None, - metavar='', - help="IP group name or ID (see 'nova ipgroup-list').") -@utils.arg('--meta', - metavar="", - action='append', - default=[], - help="Record arbitrary key/value metadata. "\ - "May be give multiple times.") -@utils.arg('--file', - metavar="", - action='append', - dest='files', - default=[], - help="Store arbitrary files from locally to "\ - "on the new server. You may store up to 5 files.") -@utils.arg('--key', - metavar='', - nargs='?', - const=AUTO_KEY, - help="Key the server with an SSH keypair. "\ - "Looks in ~/.ssh for a key, "\ - "or takes an explicit to one.") -@utils.arg('account', metavar='', help='Account to build this'\ - ' server for') -@utils.arg('name', metavar='', help='Name for the new server') -def do_boot_for_account(cs, args): - """Boot a new server in an account.""" - name, image, flavor, ipgroup, metadata, files, reservation_id, \ - min_count, max_count = _boot(cs, args) - - server = cs.accounts.create_instance_for(args.account, args.name, - image, flavor, - ipgroup=ipgroup, - meta=metadata, - files=files) - utils.print_dict(server._info) - - -@utils.arg('--flavor', - default=None, - type=int, - metavar='', - help="Flavor ID (see 'nova flavors'). "\ - "Defaults to 256MB RAM instance.") -@utils.arg('--image', - default=None, - type=int, - metavar='', - help="Image ID (see 'nova images'). "\ - "Defaults to Ubuntu 10.04 LTS.") -@utils.arg('--ipgroup', - default=None, - metavar='', - help="IP group name or ID (see 'nova ipgroup-list').") -@utils.arg('--meta', - metavar="", - action='append', - default=[], - help="Record arbitrary key/value metadata. "\ - "May be give multiple times.") -@utils.arg('--file', - metavar="", - action='append', - dest='files', - default=[], - help="Store arbitrary files from locally to "\ - "on the new server. You may store up to 5 files.") -@utils.arg('--key', - metavar='', - nargs='?', - const=AUTO_KEY, - help="Key the server with an SSH keypair. "\ - "Looks in ~/.ssh for a key, "\ - "or takes an explicit to one.") -@utils.arg('--reservation_id', - default=None, - metavar='', - help="Reservation ID (a UUID). "\ - "If unspecified will be generated by the server.") -@utils.arg('--min_instances', - default=None, - type=int, - metavar='', - help="The minimum number of instances to build. "\ - "Defaults to 1.") -@utils.arg('--max_instances', - default=None, - type=int, - metavar='', - help="The maximum number of instances to build. "\ - "Defaults to 'min_instances' setting.") -@utils.arg('name', metavar='', help='Name for the new server') -def do_zone_boot(cs, args): - """Boot a new server, potentially across Zones.""" - reservation_id = args.reservation_id - min_count = args.min_instances - max_count = args.max_instances - name, image, flavor, ipgroup, metadata, \ - files, reservation_id, min_count, max_count = \ - _boot(cs, args, - reservation_id=reservation_id, - min_count=min_count, - max_count=max_count) - - reservation_id = cs.zones.boot(args.name, image, flavor, - ipgroup=ipgroup, - meta=metadata, - files=files, - reservation_id=reservation_id, - min_count=min_count, - max_count=max_count) - print "Reservation ID=", reservation_id - - -def _translate_flavor_keys(collection): - convert = [('ram', 'memory_mb'), ('disk', 'local_gb')] - for item in collection: - keys = item.__dict__.keys() - for from_key, to_key in convert: - if from_key in keys and to_key not in keys: - setattr(item, to_key, item._info[from_key]) - - -def do_flavor_list(cs, args): - """Print a list of available 'flavors' (sizes of servers).""" - flavors = cs.flavors.list() - _translate_flavor_keys(flavors) - utils.print_list(flavors, [ - 'ID', - 'Name', - 'Memory_MB', - 'Swap', - 'Local_GB', - 'VCPUs', - 'RXTX_Factor']) - - -def do_image_list(cs, args): - """Print a list of available images to boot from.""" - server_list = {} - for server in cs.servers.list(): - server_list[server.id] = server.name - image_list = cs.images.list() - for i in range(len(image_list)): - if hasattr(image_list[i], 'serverId'): - image_list[i].serverId = server_list[image_list[i].serverId] + \ - ' (' + str(image_list[i].serverId) + ')' - utils.print_list(image_list, ['ID', 'Name', 'serverId', 'Status']) - - -@utils.arg('server', metavar='', help='Name or ID of server.') -@utils.arg('name', metavar='', help='Name of snapshot.') -def do_image_create(cs, args): - """Create a new image by taking a snapshot of a running server.""" - server = _find_server(cs, args.server) - image = cs.images.create(server, args.name) - utils.print_dict(image._info) - - -@utils.arg('image', metavar='', help='Name or ID of image.') -def do_image_delete(cs, args): - """ - Delete an image. - - It should go without saying, but you can only delete images you - created. - """ - image = _find_image(cs, args.image) - image.delete() - - -@utils.arg('server', metavar='', help='Name or ID of server.') -@utils.arg('group', metavar='', help='Name or ID of group.') -@utils.arg('address', metavar='
', help='IP address to share.') -def do_ip_share(cs, args): - """Share an IP address from the given IP group onto a server.""" - server = _find_server(cs, args.server) - group = _find_ipgroup(cs, args.group) - server.share_ip(group, args.address) - - -@utils.arg('server', metavar='', help='Name or ID of server.') -@utils.arg('address', metavar='
', - help='Shared IP address to remove from the server.') -def do_ip_unshare(cs, args): - """Stop sharing an given address with a server.""" - server = _find_server(cs, args.server) - server.unshare_ip(args.address) - - -def do_ipgroup_list(cs, args): - """Show IP groups.""" - def pretty_server_list(ipgroup): - return ", ".join(cs.servers.get(id).name - for id in ipgroup.servers) - - utils.print_list(cs.ipgroups.list(), - fields=['ID', 'Name', 'Server List'], - formatters={'Server List': pretty_server_list}) - - -@utils.arg('group', metavar='', help='Name or ID of group.') -def do_ipgroup_show(cs, args): - """Show details about a particular IP group.""" - group = _find_ipgroup(cs, args.group) - utils.print_dict(group._info) - - -@utils.arg('name', metavar='', help='What to name this new group.') -@utils.arg('server', metavar='', nargs='?', - help='Server (name or ID) to make a member of this new group.') -def do_ipgroup_create(cs, args): - """Create a new IP group.""" - if args.server: - server = _find_server(cs, args.server) - else: - server = None - group = cs.ipgroups.create(args.name, server) - utils.print_dict(group._info) - - -@utils.arg('group', metavar='', help='Name or ID of group.') -def do_ipgroup_delete(cs, args): - """Delete an IP group.""" - _find_ipgroup(cs, args.group).delete() - - -@utils.arg('--fixed_ip', - dest='fixed_ip', - metavar='', - default=None, - help='Only match against fixed IP.') -@utils.arg('--reservation_id', - dest='reservation_id', - metavar='', - default=None, - help='Only return instances that match reservation_id.') -@utils.arg('--recurse_zones', - dest='recurse_zones', - metavar='<0|1>', - nargs='?', - type=int, - const=1, - default=0, - help='Recurse through all zones if set.') -@utils.arg('--ip', - dest='ip', - metavar='', - default=None, - help='Search with regular expression match by IP address') -@utils.arg('--ip6', - dest='ip6', - metavar='', - default=None, - help='Search with regular expression match by IPv6 address') -@utils.arg('--name', - dest='name', - metavar='', - default=None, - help='Search with regular expression match by name') -@utils.arg('--instance_name', - dest='instance_name', - metavar='', - default=None, - help='Search with regular expression match by instance name') -@utils.arg('--status', - dest='status', - metavar='', - default=None, - help='Search by server status') -@utils.arg('--flavor', - dest='flavor', - metavar='', - type=int, - default=None, - help='Search by flavor ID') -@utils.arg('--image', - dest='image', - type=int, - metavar='', - default=None, - help='Search by image ID') -@utils.arg('--host', - dest='host', - metavar='', - default=None, - help="Search by instances by hostname to which they are assigned") -def do_list(cs, args): - """List active servers.""" - recurse_zones = args.recurse_zones - search_opts = { - 'reservation_id': args.reservation_id, - 'fixed_ip': args.fixed_ip, - 'recurse_zones': recurse_zones, - 'ip': args.ip, - 'ip6': args.ip6, - 'name': args.name, - 'image': args.image, - 'flavor': args.flavor, - 'status': args.status, - 'host': args.host, - 'instance_name': args.instance_name} - if recurse_zones: - to_print = ['UUID', 'Name', 'Status', 'Public IP', 'Private IP'] - else: - to_print = ['ID', 'Name', 'Status', 'Public IP', 'Private IP'] - utils.print_list(cs.servers.list(search_opts=search_opts), - to_print) - - -@utils.arg('--hard', - dest='reboot_type', - action='store_const', - const=servers.REBOOT_HARD, - default=servers.REBOOT_SOFT, - help='Perform a hard reboot (instead of a soft one).') -@utils.arg('server', metavar='', help='Name or ID of server.') -def do_reboot(cs, args): - """Reboot a server.""" - _find_server(cs, args.server).reboot(args.reboot_type) - - -@utils.arg('server', metavar='', help='Name or ID of server.') -@utils.arg('image', metavar='', help="Name or ID of new image.") -def do_rebuild(cs, args): - """Shutdown, re-image, and re-boot a server.""" - server = _find_server(cs, args.server) - image = _find_image(cs, args.image) - server.rebuild(image) - - -@utils.arg('server', metavar='', - help='Name (old name) or ID of server.') -@utils.arg('name', metavar='', help='New name for the server.') -def do_rename(cs, args): - """Rename a server.""" - _find_server(cs, args.server).update(name=args.name) - - -@utils.arg('server', metavar='', help='Name or ID of server.') -@utils.arg('flavor', metavar='', help="Name or ID of new flavor.") -def do_resize(cs, args): - """Resize a server.""" - server = _find_server(cs, args.server) - flavor = _find_flavor(cs, args.flavor) - server.resize(flavor) - - -@utils.arg('server', metavar='', help='Name or ID of server.') -@utils.arg('name', metavar='', help='Name of snapshot.') -@utils.arg('backup_type', metavar='', help='type of backup') -@utils.arg('rotation', type=int, metavar='', - help="Number of backups to retain. Used for backup image_type.") -def do_backup(cs, args): - """Backup a server.""" - server = _find_server(cs, args.server) - server.backup(args.name, args.backup_type, args.rotation) - - -@utils.arg('server', metavar='', help='Name or ID of server.') -def do_migrate(cs, args): - """Migrate a server.""" - _find_server(cs, args.server).migrate() - - -@utils.arg('server', metavar='', help='Name or ID of server.') -def do_pause(cs, args): - """Pause a server.""" - _find_server(cs, args.server).pause() - - -@utils.arg('server', metavar='', help='Name or ID of server.') -def do_unpause(cs, args): - """Unpause a server.""" - _find_server(cs, args.server).unpause() - - -@utils.arg('server', metavar='', help='Name or ID of server.') -def do_suspend(cs, args): - """Suspend a server.""" - _find_server(cs, args.server).suspend() - - -@utils.arg('server', metavar='', help='Name or ID of server.') -def do_resume(cs, args): - """Resume a server.""" - _find_server(cs, args.server).resume() - - -@utils.arg('server', metavar='', help='Name or ID of server.') -def do_rescue(cs, args): - """Rescue a server.""" - _find_server(cs, args.server).rescue() - - -@utils.arg('server', metavar='', help='Name or ID of server.') -def do_unrescue(cs, args): - """Unrescue a server.""" - _find_server(cs, args.server).unrescue() - - -@utils.arg('server', metavar='', help='Name or ID of server.') -def do_diagnostics(cs, args): - """Retrieve server diagnostics.""" - utils.print_dict(cs.servers.diagnostics(args.server)[1]) - - -@utils.arg('server', metavar='', help='Name or ID of server.') -def do_actions(cs, args): - """Retrieve server actions.""" - utils.print_list( - cs.servers.actions(args.server), - ["Created_At", "Action", "Error"]) - - -@utils.arg('server', metavar='', help='Name or ID of server.') -def do_resize_confirm(cs, args): - """Confirm a previous resize.""" - _find_server(cs, args.server).confirm_resize() - - -@utils.arg('server', metavar='', help='Name or ID of server.') -def do_resize_revert(cs, args): - """Revert a previous resize (and return to the previous VM).""" - _find_server(cs, args.server).revert_resize() - - -@utils.arg('server', metavar='', help='Name or ID of server.') -def do_root_password(cs, args): - """ - Change the root password for a server. - """ - server = _find_server(cs, args.server) - p1 = getpass.getpass('New password: ') - p2 = getpass.getpass('Again: ') - if p1 != p2: - raise exceptions.CommandError("Passwords do not match.") - server.update(password=p1) - - -@utils.arg('server', metavar='', help='Name or ID of server.') -def do_show(cs, args): - """Show details about the given server.""" - s = _find_server(cs, args.server) - - info = s._info.copy() - addresses = info.pop('addresses') - for addrtype in addresses: - info['%s ip' % addrtype] = ', '.join(addresses[addrtype]) - - flavorId = info.get('flavorId', None) - if flavorId: - info['flavor'] = _find_flavor(cs, info.pop('flavorId')).name - imageId = info.get('imageId', None) - if imageId: - info['image'] = _find_image(cs, info.pop('imageId')).name - - utils.print_dict(info) - - -@utils.arg('server', metavar='', help='Name or ID of server.') -def do_delete(cs, args): - """Immediately shut down and delete a server.""" - _find_server(cs, args.server).delete() - - -# --zone_username is required since --username is already used. -@utils.arg('zone', metavar='', help='ID of the zone', default=None) -@utils.arg('--api_url', dest='api_url', default=None, help='New URL.') -@utils.arg('--zone_username', dest='zone_username', default=None, - help='New zone username.') -@utils.arg('--zone_password', dest='zone_password', default=None, - help='New password.') -@utils.arg('--weight_offset', dest='weight_offset', default=None, - help='Child Zone weight offset.') -@utils.arg('--weight_scale', dest='weight_scale', default=None, - help='Child Zone weight scale.') -def do_zone(cs, args): - """Show or edit a child zone. No zone arg for this zone.""" - zone = cs.zones.get(args.zone) - - # If we have some flags, update the zone - zone_delta = {} - if args.api_url: - zone_delta['api_url'] = args.api_url - if args.zone_username: - zone_delta['username'] = args.zone_username - if args.zone_password: - zone_delta['password'] = args.zone_password - if args.weight_offset: - zone_delta['weight_offset'] = args.weight_offset - if args.weight_scale: - zone_delta['weight_scale'] = args.weight_scale - if zone_delta: - zone.update(**zone_delta) - else: - utils.print_dict(zone._info) - - -def do_zone_info(cs, args): - """Get this zones name and capabilities.""" - zone = cs.zones.info() - utils.print_dict(zone._info) - - -@utils.arg('zone_name', metavar='', - help='Name of the child zone being added.') -@utils.arg('api_url', metavar='', help="URL for the Zone's Auth API") -@utils.arg('--zone_username', metavar='', - help='Optional Authentication username. (Default=None)', - default=None) -@utils.arg('--zone_password', metavar='', - help='Authentication password. (Default=None)', - default=None) -@utils.arg('--weight_offset', metavar='', - help='Child Zone weight offset (Default=0.0))', - default=0.0) -@utils.arg('--weight_scale', metavar='', - help='Child Zone weight scale (Default=1.0).', - default=1.0) -def do_zone_add(cs, args): - """Add a new child zone.""" - zone = cs.zones.create(args.zone_name, args.api_url, - args.zone_username, args.zone_password, - args.weight_offset, args.weight_scale) - utils.print_dict(zone._info) - - -@utils.arg('zone', metavar='', help='Name or ID of the zone') -def do_zone_delete(cs, args): - """Delete a zone.""" - cs.zones.delete(args.zone) - - -def do_zone_list(cs, args): - """List the children of a zone.""" - utils.print_list(cs.zones.list(), ['ID', 'Name', 'Is Active', \ - 'API URL', 'Weight Offset', 'Weight Scale']) - - -@utils.arg('server', metavar='', help='Name or ID of server.') -@utils.arg('network_id', metavar='', help='Network ID.') -def do_add_fixed_ip(cs, args): - """Add new IP address to network.""" - server = _find_server(cs, args.server) - server.add_fixed_ip(args.network_id) - - -@utils.arg('server', metavar='', help='Name or ID of server.') -@utils.arg('address', metavar='
', help='IP Address.') -def do_remove_fixed_ip(cs, args): - """Remove an IP address from a server.""" - server = _find_server(cs, args.server) - server.remove_fixed_ip(args.address) - - -def _find_server(cs, server): - """Get a server by name or ID.""" - return utils.find_resource(cs.servers, server) - - -def _find_ipgroup(cs, group): - """Get an IP group by name or ID.""" - return utils.find_resource(cs.ipgroups, group) - - -def _find_image(cs, image): - """Get an image by name or ID.""" - return utils.find_resource(cs.images, image) - - -def _find_flavor(cs, flavor): - """Get a flavor by name, ID, or RAM size.""" - try: - return utils.find_resource(cs.flavors, flavor) - except exceptions.NotFound: - return cs.flavors.find(ram=flavor) diff --git a/v1_0/zones.py b/v1_0/zones.py deleted file mode 100644 index 6e75ed2a..00000000 --- a/v1_0/zones.py +++ /dev/null @@ -1,199 +0,0 @@ -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Zone interface. -""" - -from novaclient import base -import base as local_base - - -class Weighting(base.Resource): - def __init__(self, manager, info, loaded=False): - self.name = "n/a" - super(Weighting, self).__init__(manager, info, loaded) - - def __repr__(self): - return "" % self.name - - def to_dict(self): - """Return the original info setting, which is a dict.""" - return self._info - - -class Zone(base.Resource): - def __init__(self, manager, info, loaded=False): - self.name = "n/a" - self.is_active = "n/a" - self.capabilities = "n/a" - super(Zone, self).__init__(manager, info, loaded) - - def __repr__(self): - return "" % self.api_url - - def delete(self): - """ - Delete a child zone. - """ - self.manager.delete(self) - - def update(self, api_url=None, username=None, password=None, - weight_offset=None, weight_scale=None): - """ - Update the name for this child zone. - - :param api_url: Update the child zone's API URL. - :param username: Update the child zone's username. - :param password: Update the child zone's password. - :param weight_offset: Update the child zone's weight offset. - :param weight_scale: Update the child zone's weight scale. - """ - self.manager.update(self, api_url, username, password, - weight_offset, weight_scale) - - -class ZoneManager(local_base.BootingManagerWithFind): - resource_class = Zone - - def info(self): - """ - Get info on this zone. - - :rtype: :class:`Zone` - """ - return self._get("/zones/info", "zone") - - def get(self, zone): - """ - Get a child zone. - - :param server: ID of the :class:`Zone` to get. - :rtype: :class:`Zone` - """ - return self._get("/zones/%s" % base.getid(zone), "zone") - - def list(self, detailed=True): - """ - Get a list of child zones. - :rtype: list of :class:`Zone` - """ - detail = "" - if detailed: - detail = "/detail" - return self._list("/zones%s" % detail, "zones") - - def create(self, zone_name, api_url, username, password, - weight_offset=0.0, weight_scale=1.0): - """ - Create a new child zone. - - :param zone_name: The child zone's name. - :param api_url: The child zone's auth URL. - :param username: The child zone's username. - :param password: The child zone's password. - :param weight_offset: The child zone's weight offset. - :param weight_scale: The child zone's weight scale. - """ - body = {"zone": { - "name": zone_name, - "api_url": api_url, - "username": username, - "password": password, - "weight_offset": weight_offset, - "weight_scale": weight_scale - }} - - return self._create("/zones", body, "zone") - - def boot(self, name, image, flavor, ipgroup=None, meta=None, files=None, - zone_blob=None, reservation_id=None, min_count=None, - max_count=None): - """ - Create (boot) a new server while being aware of Zones. - - :param name: Something to name the server. - :param image: The :class:`Image` to boot with. - :param flavor: The :class:`Flavor` to boot onto. - :param ipgroup: An initial :class:`IPGroup` for this server. - :param meta: A dict of arbitrary key/value metadata to store for this - server. A maximum of five entries is allowed, and both - keys and values must be 255 characters or less. - :param files: A dict of files to overrwrite on the server upon boot. - Keys are file names (i.e. ``/etc/passwd``) and values - are the file contents (either as a string or as a - file-like object). A maximum of five entries is allowed, - and each file must be 10k or less. - :param zone_blob: a single (encrypted) string which is used internally - by Nova for routing between Zones. Users cannot populate - this field. - :param reservation_id: a UUID for the set of servers being requested. - :param min_count: minimum number of servers to create. - :param max_count: maximum number of servers to create. - """ - if not min_count: - min_count = 1 - if not max_count: - max_count = min_count - return self._boot("/zones/boot", "reservation_id", name, image, flavor, - ipgroup=ipgroup, meta=meta, files=files, - zone_blob=zone_blob, reservation_id=reservation_id, - return_raw=True, min_count=min_count, - max_count=max_count) - - def select(self, *args, **kwargs): - """ - Given requirements for a new instance, select hosts - in this zone that best match those requirements. - """ - # 'specs' may be passed in as None, so change to an empty string. - specs = kwargs.get("specs") or "" - url = "/zones/select" - weighting_list = self._list(url, "weights", Weighting, body=specs) - return [wt.to_dict() for wt in weighting_list] - - def delete(self, zone): - """ - Delete a child zone. - """ - self._delete("/zones/%s" % base.getid(zone)) - - def update(self, zone, api_url=None, username=None, password=None, - weight_offset=None, weight_scale=None): - """ - Update the name or the api_url for a zone. - - :param zone: The :class:`Zone` (or its ID) to update. - :param api_url: Update the API URL. - :param username: Update the username. - :param password: Update the password. - :param weight_offset: Update the child zone's weight offset. - :param weight_scale: Update the child zone's weight scale. - """ - - body = {"zone": {}} - if api_url: - body["zone"]["api_url"] = api_url - if username: - body["zone"]["username"] = username - if password: - body["zone"]["password"] = password - if weight_offset: - body["zone"]["weight_offset"] = weight_offset - if weight_scale: - body["zone"]["weight_scale"] = weight_scale - if not len(body["zone"]): - return - self._update("/zones/%s" % base.getid(zone), body) diff --git a/vmdatabase.py b/vmdatabase.py deleted file mode 100644 index 5697d338..00000000 --- a/vmdatabase.py +++ /dev/null @@ -1,421 +0,0 @@ -#!/usr/bin/env python - -# Keep track of VMs used by the devstack gate test. - -# Copyright (C) 2011-2012 OpenStack LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import time - -# States: -# The cloud provider is building this machine. We have an ID, but it's -# not ready for use. -BUILDING = 1 -# The machine is ready for use. -READY = 2 -# This can mean in-use, or used but complete. We don't actually need to -# distinguish between those states -- we'll just delete a machine 24 hours -# after it transitions into the USED state. -USED = 3 -# An error state, should just try to delete it. -ERROR = 4 -# Keep this machine indefinitely -HOLD = 5 -# Delete this machine immediately (probably a used machine) -DELETE = 6 - -# Possible Jenkins results -RESULT_SUCCESS = 1 -RESULT_FAILURE = 2 -RESULT_TIMEOUT = 3 - -from sqlalchemy import Table, Column, Boolean, Integer, String, \ - MetaData, ForeignKey, \ - create_engine, and_ -from sqlalchemy.orm import mapper, relation -from sqlalchemy.orm.session import Session, sessionmaker - -metadata = MetaData() -provider_table = Table('provider', metadata, - Column('id', Integer, primary_key=True), - Column('name', String(255), index=True, unique=True), - # Max total number of servers for this provider - Column('max_servers', Integer), - # May we give failed vms from this provider to developers? - Column('giftable', Boolean), - # 1.0 or 1.1 - Column('nova_api_version', String(8)), - # novaclient doesn't discover this itself - Column('nova_rax_auth', Boolean), - Column('nova_username', String(255)), - Column('nova_api_key', String(255)), - # Authentication URL - Column('nova_auth_url', String(255)), - # Project id to use at authn - Column('nova_project_id', String(255)), - # endpoint selection: service type (Null for default) - Column('nova_service_type', String(255)), - # endpoint selection: service region (Null for default) - Column('nova_service_region', String(255)), - # endpoint selection: Endpoint name (Null for default) - Column('nova_service_name', String(255)), - ) -base_image_table = Table('base_image', metadata, - Column('id', Integer, primary_key=True), - Column('provider_id', Integer, ForeignKey('provider.id'), - index=True, nullable=False), - # Image name (oneiric, precise, etc). - Column('name', String(255)), - # Provider assigned id for this image - Column('external_id', String(255)), - # Min number of servers to keep ready for this provider/image - Column('min_ready', Integer), - # amount of ram to select for servers with this image - Column('min_ram', Integer), - #active? - ) -snapshot_image_table = Table('snapshot_image', metadata, - Column('id', Integer, primary_key=True), - Column('name', String(255)), - Column('base_image_id', Integer, ForeignKey('base_image.id'), - index=True, nullable=False), - # Version indicator (timestamp) - Column('version', Integer), - # Provider assigned id for this image - Column('external_id', String(255)), - # Provider assigned id of the server used to create the snapshot - Column('server_external_id', String(255)), - # One of the above values - Column('state', Integer), - # Time of last state change - Column('state_time', Integer), - ) -machine_table = Table('machine', metadata, - Column('id', Integer, primary_key=True), - Column('base_image_id', Integer, ForeignKey('base_image.id'), - index=True, nullable=False), - # Provider assigned id for this machine - Column('external_id', String(255)), - # Machine name - Column('name', String(255)), - # Jenkins node name - Column('jenkins_name', String(255)), - # Primary IP address - Column('ip', String(255)), - # Username if ssh keys have been installed, or NULL - Column('user', String(255)), - # One of the above values - Column('state', Integer), - # Time of last state change - Column('state_time', Integer), - ) -result_table = Table('result', metadata, - Column('id', Integer, primary_key=True), - Column('base_image_id', Integer, ForeignKey('base_image.id'), - index=True, nullable=False), - # Not a FK so that machines can be deleted - Column('machine_id', Integer), - Column('jenkins_job_name', String(255)), - Column('jenkins_build_number', Integer), - Column('gerrit_change_number', Integer), - Column('gerrit_patchset_number', Integer), - # Time that the job was started - Column('start_time', Integer), - # Time the job finished - Column('end_time', Integer), - # Result of job - Column('result', Integer), - ) - - -class Provider(object): - def __init__(self, name, driver, username, api_key, giftable): - self.name = name - self.driver = driver - self.username = username - self.api_key = api_key - self.giftable = giftable - - def delete(self): - session = Session.object_session(self) - session.delete(self) - session.commit() - - def newBaseImage(self, *args, **kwargs): - new = BaseImage(*args, **kwargs) - new.provider = self - session = Session.object_session(self) - session.commit() - return new - - def getBaseImage(self, name): - session = Session.object_session(self) - return session.query(BaseImage).filter(and_( - base_image_table.c.name == name, - base_image_table.c.provider_id == self.id)).first() - - def _machines(self): - session = Session.object_session(self) - return session.query(Machine).filter(and_( - machine_table.c.base_image_id == base_image_table.c.id, - base_image_table.c.provider_id == self.id)).order_by( - machine_table.c.state_time) - - @property - def machines(self): - return self._machines().all() - - @property - def building_machines(self): - return self._machines().filter(machine_table.c.state == BUILDING).all() - - @property - def ready_machines(self): - return self._machines().filter(machine_table.c.state == READY).all() - - -class BaseImage(object): - def __init__(self, name, external_id): - self.name = name - self.external_id = external_id - - def delete(self): - session = Session.object_session(self) - session.delete(self) - session.commit() - - def newSnapshotImage(self, *args, **kwargs): - new = SnapshotImage(*args, **kwargs) - new.base_image = self - session = Session.object_session(self) - session.commit() - return new - - def newMachine(self, *args, **kwargs): - new = Machine(*args, **kwargs) - new.base_image = self - session = Session.object_session(self) - session.commit() - return new - - @property - def ready_snapshot_images(self): - session = Session.object_session(self) - return session.query(SnapshotImage).filter(and_( - snapshot_image_table.c.base_image_id == self.id, - snapshot_image_table.c.state == READY)).order_by( - snapshot_image_table.c.version).all() - - @property - def current_snapshot(self): - if not self.ready_snapshot_images: - return None - return self.ready_snapshot_images[-1] - - def _machines(self): - session = Session.object_session(self) - return session.query(Machine).filter( - machine_table.c.base_image_id == self.id).order_by( - machine_table.c.state_time) - - @property - def building_machines(self): - return self._machines().filter(machine_table.c.state == BUILDING).all() - - @property - def ready_machines(self): - return self._machines().filter(machine_table.c.state == READY).all() - - -class SnapshotImage(object): - def __init__(self, name, version, external_id, server_external_id, - state=BUILDING): - self.name = name - self.version = version - self.external_id = external_id - self.server_external_id = server_external_id - self.state = state - - def delete(self): - session = Session.object_session(self) - session.delete(self) - session.commit() - - @property - def state(self): - return self._state - - @state.setter - def state(self, state): - self._state = state - self.state_time = int(time.time()) - session = Session.object_session(self) - if session: - session.commit() - - -class Machine(object): - def __init__(self, name, external_id, ip=None, user=None, - state=BUILDING): - self.name = name - self.external_id = external_id - self.ip = ip - self.user = user - self.state = state - - def delete(self): - session = Session.object_session(self) - session.delete(self) - session.commit() - - @property - def state(self): - return self._state - - @state.setter - def state(self, state): - self._state = state - self.state_time = int(time.time()) - session = Session.object_session(self) - if session: - session.commit() - - def newResult(self, jenkins_job_name, jenkins_build_number, - gerrit_change_number, gerrit_patchset_number): - new = Result(self.id, jenkins_job_name, jenkins_build_number, - gerrit_change_number, gerrit_patchset_number, time.time()) - new.base_image = self.base_image - session = Session.object_session(self) - session.commit() - return new - - -class Result(object): - def __init__(self, machine_id, jenkins_job_name, jenkins_build_number, - gerrit_change_number, gerrit_patchset_number, - start_time, end_time=None, result=None): - self.machine_id = machine_id - self.jenkins_job_name = jenkins_job_name - self.jenkins_build_number = jenkins_build_number - self.gerrit_change_number = gerrit_change_number - self.gerrit_patchset_number = gerrit_patchset_number - self.start_time = start_time - self.end_time = end_time - self.result = result - - def setResult(self, result): - self.result = result - self.end_time = time.time() - session = Session.object_session(self) - session.commit() - - def delete(self): - session = Session.object_session(self) - session.delete(self) - session.commit() - - -mapper(Result, result_table) - -mapper(Machine, machine_table, properties=dict( - _state=machine_table.c.state, - )) - -mapper(SnapshotImage, snapshot_image_table, properties=dict( - _state=snapshot_image_table.c.state, - )) - -mapper(BaseImage, base_image_table, properties=dict( - snapshot_images=relation(SnapshotImage, - order_by=snapshot_image_table.c.version, - cascade='all, delete-orphan', - backref='base_image'), - machines=relation(Machine, - order_by=machine_table.c.state_time, - cascade='all, delete-orphan', - backref='base_image'), - results=relation(Result, - order_by=result_table.c.start_time, - cascade='all, delete-orphan', - backref='base_image'))) - -mapper(Provider, provider_table, properties=dict( - base_images=relation(BaseImage, - order_by=base_image_table.c.name, - cascade='all, delete-orphan', - backref='provider'))) - - -class VMDatabase(object): - def __init__(self, path=os.path.expanduser("~/vm.db")): - engine = create_engine('sqlite:///%s' % path, echo=False) - metadata.create_all(engine) - Session = sessionmaker(bind=engine, autoflush=True, autocommit=False) - self.session = Session() - - def print_state(self): - for provider in self.getProviders(): - print 'Provider:', provider.name - for base_image in provider.base_images: - print ' Base image:', base_image.name - for snapshot_image in base_image.snapshot_images: - print ' Snapshot:', snapshot_image.name, \ - snapshot_image.state - for machine in base_image.machines: - print ' Machine:', machine.id, machine.name, \ - machine.state, machine.state_time, machine.ip - - def abort(self): - self.session.rollback() - - def commit(self): - self.session.commit() - - def delete(self, obj): - self.session.delete(obj) - - def getProviders(self): - return self.session.query(Provider).all() - - def getProvider(self, name): - return self.session.query(Provider).filter_by(name=name)[0] - - def getResult(self, id): - return self.session.query(Result).filter_by(id=id)[0] - - def getMachine(self, id): - return self.session.query(Machine).filter_by(id=id)[0] - - def getMachineByJenkinsName(self, name): - return self.session.query(Machine).filter_by(jenkins_name=name)[0] - - def getMachineForUse(self, image_name): - """Atomically find a machine that is ready for use, and update - its state.""" - for machine in self.session.query(Machine).filter( - machine_table.c.state == READY).order_by( - machine_table.c.state_time): - if machine.base_image.name == image_name: - machine.state = USED - self.commit() - return machine - raise Exception("No machine found for image %s" % image_name) - - -if __name__ == '__main__': - db = VMDatabase() - db.print_state()