diff --git a/devstack-vm-delete.py b/devstack-vm-delete.py index 6431f869..6d3501b3 100755 --- a/devstack-vm-delete.py +++ b/devstack-vm-delete.py @@ -34,14 +34,19 @@ NODE_NAME = sys.argv[1] UPSTREAM_BUILD_URL=os.environ.get('UPSTREAM_BUILD_URL', '') UPSTREAM_JOB_NAME=os.environ.get('UPSTREAM_JOB_NAME', '') UPSTREAM_BRANCH=os.environ.get('UPSTREAM_BRANCH', '') - +BUILD_URL=os.environ.get('BUILD_URL', '') def main(): db = vmdatabase.VMDatabase() machine = db.getMachineByJenkinsName(NODE_NAME) if machine.state != vmdatabase.HOLD: + utils.log.debug("Set deleted ID: %s old state: %s build: %s" % ( + machine.id, machine.state, BUILD_URL)) machine.state = vmdatabase.DELETE + else: + utils.log.debug("Hold ID: %s old state: %s build: %s" % ( + machine.id, machine.state, BUILD_URL)) try: utils.update_stats(machine.base_image.provider) diff --git a/devstack-vm-inprogress.py b/devstack-vm-inprogress.py index 4796a093..29c583b5 100755 --- a/devstack-vm-inprogress.py +++ b/devstack-vm-inprogress.py @@ -35,8 +35,10 @@ NODE_NAME = sys.argv[1] DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG', os.path.expanduser( '~/devstack-gate-secure.conf')) +SKIP_DEVSTACK_GATE_JENKINS = os.environ.get('SKIP_DEVSTACK_GATE_JENKINS', None) +BUILD_URL=os.environ.get('BUILD_URL', '') -LABEL_RE = re.compile(r'') +LABEL_RE = re.compile(r'') def main(): @@ -45,19 +47,39 @@ def main(): config = ConfigParser.ConfigParser() config.read(DEVSTACK_GATE_SECURE_CONFIG) - jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'), - config.get('jenkins', 'user'), - config.get('jenkins', 'apikey')) - jenkins.get_info() + if not SKIP_DEVSTACK_GATE_JENKINS: + jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'), + config.get('jenkins', 'user'), + config.get('jenkins', 'apikey')) + jenkins.get_info() + else: + jenkins = None machine = db.getMachineByJenkinsName(NODE_NAME) + utils.log.debug("Used ID: %s old state: %s build:%s" % ( + machine.id, machine.state, BUILD_URL)) + machine.state = vmdatabase.USED - if machine.jenkins_name: - if jenkins.node_exists(machine.jenkins_name): - config = jenkins.get_node_config(machine.jenkins_name) - config = LABEL_RE.sub('', config) - jenkins.reconfig_node(machine.jenkins_name, config) + if jenkins: + if machine.jenkins_name: + if jenkins.node_exists(machine.jenkins_name): + config = jenkins.get_node_config(machine.jenkins_name) + old = None + m = LABEL_RE.search(config) + if m: + old = m.group(1) + config = LABEL_RE.sub('', config) + for i in range(3): + try: + jenkins.reconfig_node(machine.jenkins_name, config) + except: + if i==2: + utils.log.exception("Unable to relabel ID: %s" % machine.id) + raise + time.sleep(5) + utils.log.debug("Relabeled ID: %s old label: %s new label: %s" % ( + machine.id, old, 'devstack-used')) utils.update_stats(machine.base_image.provider) diff --git a/devstack-vm-launch.py b/devstack-vm-launch.py index 2e87433c..65714ba7 100755 --- a/devstack-vm-launch.py +++ b/devstack-vm-launch.py @@ -81,6 +81,8 @@ def launch_node(client, snap_image, image, flavor, last_name): print " id: %s" % (server.id) print " name: %s" % (name) print + utils.log.debug("Launching ID: %s name: %s provider ID: %s" % + (machine.id, name, server.id)) return server, machine @@ -137,6 +139,7 @@ def check_machine(jenkins, client, machine, error_counts): create_jenkins_node(jenkins, machine) print "Machine %s is ready" % machine.id machine.state = vmdatabase.READY + utils.log.debug("Online ID: %s" % machine.id) return elif not server.status.startswith('BUILD'): count = error_counts.get(machine.id, 0) @@ -220,6 +223,7 @@ def main(): except: traceback.print_exc() print "Abandoning machine %s" % machine.id + utils.log.exception("Abandoning ID: %s" % machine.id) machine.state = vmdatabase.ERROR error = True db.commit() diff --git a/devstack-vm-reap.py b/devstack-vm-reap.py index ce44fc22..63406200 100755 --- a/devstack-vm-reap.py +++ b/devstack-vm-reap.py @@ -63,12 +63,14 @@ def delete_machine(jenkins, client, machine): # If we have deleted a server, don't believe it. Instead, wait for # the next run of the script and only if the server doesn't exist, # delete it from Jenkins and the DB. + utils.log.debug("Delete ID: %s" % machine.id) return if jenkins: if machine.jenkins_name: if jenkins.node_exists(machine.jenkins_name): jenkins.delete_node(machine.jenkins_name) + utils.log.debug("Delete jenkins node ID: %s" % machine.id) machine.delete() diff --git a/utils.py b/utils.py index f237e803..bc22f407 100644 --- a/utils.py +++ b/utils.py @@ -28,9 +28,17 @@ import paramiko import socket from sshclient import SSHClient from statsd import statsd +import logging +import logging.handlers import vmdatabase +log = logging.getLogger('devstack-gate') +log.setLevel(logging.DEBUG) +handler = logging.handlers.SysLogHandler(address = '/dev/log') +handler.setFormatter(logging.Formatter("devstack-gate: %(message)s")) +log.addHandler(handler) + def iterate_timeout(max_seconds, purpose): start = time.time() count = 0