diff --git a/glance/store/scrubber.py b/glance/store/scrubber.py index 950a4277..4d975517 100644 --- a/glance/store/scrubber.py +++ b/glance/store/scrubber.py @@ -121,10 +121,9 @@ class Scrubber(object): delete_work.append((id, uri, now)) LOG.info(_("Deleting %s images") % len(delete_work)) - pool.starmap(self._delete, delete_work) - # NOTE(bourke): When not running as a daemon, a slight pause is needed - # to allow the starmap to begin it's work. - eventlet.sleep(0.1) + # NOTE(bourke): The starmap must be iterated to do work + for job in pool.starmap(self._delete, delete_work): + pass if self.cleanup: self._cleanup(pool) @@ -187,7 +186,9 @@ class Scrubber(object): now)) LOG.info(_("Deleting %s images") % len(delete_work)) - pool.starmap(self._delete, delete_work) + # NOTE(bourke): The starmap must be iterated to do work + for job in pool.starmap(self._delete, delete_work): + pass def read_queue_file(file_path): diff --git a/glance/tests/functional/__init__.py b/glance/tests/functional/__init__.py index 90819898..46468219 100644 --- a/glance/tests/functional/__init__.py +++ b/glance/tests/functional/__init__.py @@ -193,10 +193,10 @@ class ApiServer(Server): """ def __init__(self, test_dir, port, policy_file, delayed_delete=False, - pid_file=None): + pid_file=None, **kwargs): super(ApiServer, self).__init__(test_dir, port) self.server_name = 'api' - self.default_store = 'file' + self.default_store = kwargs.get("default_store", "file") self.key_file = "" self.cert_file = "" self.metadata_encryption_key = "012345678901234567890123456789ab" @@ -209,10 +209,15 @@ class ApiServer(Server): self.s3_store_secret_key = "" self.s3_store_bucket = "" self.s3_store_bucket_url_format = "" - self.swift_store_auth_address = "" - self.swift_store_user = "" - self.swift_store_key = "" - self.swift_store_container = "" + self.swift_store_auth_version = kwargs.get("swift_store_auth_version", + "2") + self.swift_store_auth_address = kwargs.get("swift_store_auth_address", + "") + self.swift_store_user = kwargs.get("swift_store_user", "") + self.swift_store_key = kwargs.get("swift_store_key", "") + self.swift_store_container = kwargs.get("swift_store_container", "") + self.swift_store_create_container_on_put = kwargs.get( + "swift_store_create_container_on_put", "True") self.swift_store_large_object_size = 5 * 1024 self.swift_store_large_object_chunk_size = 200 self.swift_store_multi_tenant = False @@ -255,10 +260,12 @@ s3_store_access_key = %(s3_store_access_key)s s3_store_secret_key = %(s3_store_secret_key)s s3_store_bucket = %(s3_store_bucket)s s3_store_bucket_url_format = %(s3_store_bucket_url_format)s +swift_store_auth_version = %(swift_store_auth_version)s swift_store_auth_address = %(swift_store_auth_address)s swift_store_user = %(swift_store_user)s swift_store_key = %(swift_store_key)s swift_store_container = %(swift_store_container)s +swift_store_create_container_on_put = %(swift_store_create_container_on_put)s swift_store_large_object_size = %(swift_store_large_object_size)s swift_store_large_object_chunk_size = %(swift_store_large_object_chunk_size)s swift_store_multi_tenant = %(swift_store_multi_tenant)s @@ -404,7 +411,7 @@ class ScrubberDaemon(Server): Server object that starts/stops/manages the Scrubber server """ - def __init__(self, test_dir, daemon=False): + def __init__(self, test_dir, daemon=False, **kwargs): # NOTE(jkoelker): Set the port to 0 since we actually don't listen super(ScrubberDaemon, self).__init__(test_dir, 0) self.server_name = 'scrubber' @@ -414,6 +421,13 @@ class ScrubberDaemon(Server): "scrubber") self.pid_file = os.path.join(self.test_dir, "scrubber.pid") self.log_file = os.path.join(self.test_dir, "scrubber.log") + self.swift_store_auth_address = kwargs.get("swift_store_auth_address", + "") + self.swift_store_user = kwargs.get("swift_store_user", "") + self.swift_store_key = kwargs.get("swift_store_key", "") + self.swift_store_container = kwargs.get("swift_store_container", "") + self.swift_store_auth_version = kwargs.get("swift_store_auth_version", + "2") self.conf_base = """[DEFAULT] verbose = %(verbose)s debug = %(debug)s @@ -423,6 +437,11 @@ wakeup_time = 2 scrubber_datadir = %(scrubber_datadir)s registry_host = 127.0.0.1 registry_port = %(registry_port)s +swift_store_auth_address = %(swift_store_auth_address)s +swift_store_user = %(swift_store_user)s +swift_store_key = %(swift_store_key)s +swift_store_container = %(swift_store_container)s +swift_store_auth_version = %(swift_store_auth_version)s """ diff --git a/glance/tests/functional/test_scrubber.py b/glance/tests/functional/test_scrubber.py index 1e419e02..82b9b12e 100644 --- a/glance/tests/functional/test_scrubber.py +++ b/glance/tests/functional/test_scrubber.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack, LLC +# Copyright 2011-2012 OpenStack, LLC # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -16,11 +16,14 @@ # under the License. import json +import os import time - import httplib2 +import nose from glance.tests import functional +from glance.tests.functional.store.test_swift import parse_config +from glance.tests.functional.store.test_swift import read_config from glance.tests.utils import execute @@ -149,3 +152,77 @@ class TestScrubber(functional.FunctionalTest): self.fail('image was never scrubbed') self.stop_servers() + + def test_scrubber_app_against_swift(self): + """ + test that the glance-scrubber script runs successfully against a swift + backend when not in daemon mode + """ + config_path = os.environ.get('GLANCE_TEST_SWIFT_CONF') + if not config_path: + msg = "GLANCE_TEST_SWIFT_CONF environ not set." + raise nose.SkipTest(msg) + + raw_config = read_config(config_path) + swift_config = parse_config(raw_config) + + self.cleanup() + self.start_servers(delayed_delete=True, daemon=False, + default_store='swift', **swift_config) + + # add an image + headers = { + 'x-image-meta-name': 'test_image', + 'x-image-meta-is_public': 'true', + 'x-image-meta-disk_format': 'raw', + 'x-image-meta-container_format': 'ovf', + 'content-type': 'application/octet-stream', + } + path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port) + http = httplib2.Http() + response, content = http.request(path, 'POST', body='XXX', + headers=headers) + # ensure the request was successful and the image is active + self.assertEqual(response.status, 201) + image = json.loads(content)['image'] + self.assertEqual('active', image['status']) + image_id = image['id'] + + # delete the image + path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port, + image_id) + http = httplib2.Http() + response, content = http.request(path, 'DELETE') + self.assertEqual(response.status, 200) + + # ensure the image is marked pending delete + response, content = http.request(path, 'HEAD') + self.assertEqual(response.status, 200) + self.assertEqual('pending_delete', response['x-image-meta-status']) + + # wait for the scrub time on the image to pass + time.sleep(self.api_server.scrub_time) + + # call the scrubber to scrub images + cmd = ("bin/glance-scrubber --config-file %s" % + self.scrubber_daemon.conf_file_name) + exitcode, out, err = execute(cmd, raise_error=False) + self.assertEqual(0, exitcode) + + # ensure the image has been successfully deleted + # NOTE(jkoelker) The build servers sometimes take longer than + # 15 seconds to scrub. Give it up to 5 min, checking + # checking every 15 seconds. When/if it flips to + # deleted, bail immediately. + for _ in xrange(3): + time.sleep(5) + response, content = http.request(path, 'HEAD') + if (response['x-image-meta-status'] == 'deleted' and + response['x-image-meta-deleted'] == 'True'): + break + else: + continue + else: + self.fail('image was never scrubbed') + + self.stop_servers()