openstack/glance
Code Issues Proposed changes

Merge "Scrubber refactor"

Browse Source
This commit is contained in:
Zuul 2018-01-12 09:43:14 +00:00 committed by Gerrit Code Review
commit bcb19c35e7
4 changed files with 40 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
glance/opts.py
View File

@ -92,10 +92,7 @@ _scrubber_opts = [
glance.common.config.common_opts,
glance.scrubber.scrubber_opts,
glance.scrubber.scrubber_cmd_opts,
glance.scrubber.scrubber_cmd_cli_opts,
glance.registry.client.registry_client_opts,
glance.registry.client.registry_client_ctx_opts,
glance.registry.registry_addr_opts))),
glance.scrubber.scrubber_cmd_cli_opts))),
]
_cache_opts = [
(None, list(itertools.chain(

142
glance/scrubber.py
View File

@ -21,14 +21,13 @@ from glance_store import exceptions as store_exceptions
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import encodeutils
import six
from glance.common import crypt
from glance.common import exception
from glance.common import timeutils
from glance import context
import glance.db as db_api
from glance.i18n import _, _LC, _LE, _LI, _LW
import glance.registry.client.v1.api as registry
LOG = logging.getLogger(__name__)
@ -100,63 +99,6 @@ Related options:
* ``wakeup_time``
* ``scrub_pool_size``
""")),
# Note: Though the conf option admin_role is used by other Glance
# service and their usage differs requiring us to have a differing
# help text here, oslo.config generator treats them as the same
# config option and would throw a DuplicateError exception in case
# of differing help texts. Hence we have the same help text for
# admin_role here and in context.py.
cfg.StrOpt('admin_role', default='admin',
help=_("""
Role used to identify an authenticated user as administrator.
Provide a string value representing a Keystone role to identify an
administrative user. Users with this role will be granted
administrative privileges. The default value for this option is
'admin'.
Possible values:
* A string value which is a valid Keystone role
Related options:
* None
""")),
cfg.BoolOpt('send_identity_headers',
default=False,
help=_("""
Send headers received from identity when making requests to
registry.
Typically, Glance registry can be deployed in multiple flavors,
which may or may not include authentication. For example,
``trusted-auth`` is a flavor that does not require the registry
service to authenticate the requests it receives. However, the
registry service may still need a user context to be populated to
serve the requests. This can be achieved by the caller
(the Glance API usually) passing through the headers it received
from authenticating with identity for the same request. The typical
headers sent are ``X-User-Id``, ``X-Tenant-Id``, ``X-Roles``,
``X-Identity-Status`` and ``X-Service-Catalog``.
Provide a boolean value to determine whether to send the identity
headers to provide tenant and user information along with the
requests to registry service. By default, this option is set to
``False``, which means that user and tenant information is not
available readily. It must be obtained by authenticating. Hence, if
this is set to ``False``, ``flavor`` must be set to value that
either includes authentication or authenticated user context.
Possible values:
* True
* False
Related options:
* flavor
""")),
]
@ -214,6 +156,7 @@ Related options:
CONF = cfg.CONF
CONF.register_opts(scrubber_opts)
CONF.import_opt('metadata_encryption_key', 'glance.common.config')
REASONABLE_DB_PAGE_SIZE = 1000
class ScrubDBQueue(object):
@ -221,26 +164,7 @@ class ScrubDBQueue(object):
def __init__(self):
self.scrub_time = CONF.scrub_time
self.metadata_encryption_key = CONF.metadata_encryption_key
registry.configure_registry_client()
registry.configure_registry_admin_creds()
admin_user = CONF.admin_user
admin_tenant = CONF.admin_tenant_name
if CONF.send_identity_headers:
# When registry is operating in trusted-auth mode
roles = [CONF.admin_role]
self.admin_context = context.RequestContext(user=admin_user,
tenant=admin_tenant,
auth_token=None,
roles=roles)
self.registry = registry.get_registry_client(self.admin_context)
else:
ctxt = context.RequestContext()
self.registry = registry.get_registry_client(ctxt)
admin_token = self.registry.auth_token
self.admin_context = context.RequestContext(user=admin_user,
tenant=admin_tenant,
auth_token=admin_token)
self.admin_context = context.get_admin_context(show_deleted=True)
def add_location(self, image_id, location):
"""Adding image location to scrub queue.
@ -261,14 +185,12 @@ class ScrubDBQueue(object):
def _get_images_page(self, marker):
filters = {'deleted': True,
'is_public': 'none',
'status': 'pending_delete'}
if marker:
return self.registry.get_images_detailed(filters=filters,
marker=marker)
else:
return self.registry.get_images_detailed(filters=filters)
return db_api.get_api().image_get_all(self.admin_context,
filters=filters,
marker=marker,
limit=REASONABLE_DB_PAGE_SIZE)
def _get_all_images(self):
"""Generator to fetch all appropriate images, paging as needed."""
@ -296,23 +218,23 @@ class ScrubDBQueue(object):
deleted_at = image.get('deleted_at')
if not deleted_at:
continue
# NOTE: Strip off microseconds which may occur after the last '.,'
# Example: 2012-07-07T19:14:34.974216
deleted_at = timeutils.isotime(deleted_at)
date_str = deleted_at.rsplit('.', 1)[0].rsplit(',', 1)[0]
delete_time = calendar.timegm(time.strptime(date_str,
"%Y-%m-%dT%H:%M:%S"))
"%Y-%m-%dT%H:%M:%SZ"))
if delete_time + self.scrub_time > time.time():
continue
for loc in image['location_data']:
for loc in image['locations']:
if loc['status'] != 'pending_delete':
continue
if self.metadata_encryption_key:
uri = crypt.urlsafe_encrypt(self.metadata_encryption_key,
loc['url'], 64)
uri = crypt.urlsafe_decrypt(self.metadata_encryption_key,
loc['url'])
else:
uri = loc['url']
@ -327,7 +249,7 @@ class ScrubDBQueue(object):
:returns: a boolean value to inform including or not
"""
try:
image = self.registry.get_image(image_id)
image = db_api.get_api().image_get(self.admin_context, image_id)
return image['status'] == 'pending_delete'
except exception.NotFound:
return False
@ -372,36 +294,9 @@ class Daemon(object):
class Scrubber(object):
def __init__(self, store_api):
LOG.info(_LI("Initializing scrubber with configuration: %s"),
six.text_type({'registry_host': CONF.registry_host,
'registry_port': CONF.registry_port}))
LOG.info(_LI("Initializing scrubber"))
self.store_api = store_api
registry.configure_registry_client()
registry.configure_registry_admin_creds()
# Here we create a request context with credentials to support
# delayed delete when using multi-tenant backend storage
admin_user = CONF.admin_user
admin_tenant = CONF.admin_tenant_name
if CONF.send_identity_headers:
# When registry is operating in trusted-auth mode
roles = [CONF.admin_role]
self.admin_context = context.RequestContext(user=admin_user,
tenant=admin_tenant,
auth_token=None,
roles=roles)
self.registry = registry.get_registry_client(self.admin_context)
else:
ctxt = context.RequestContext()
self.registry = registry.get_registry_client(ctxt)
auth_token = self.registry.auth_token
self.admin_context = context.RequestContext(user=admin_user,
tenant=admin_tenant,
auth_token=auth_token)
self.admin_context = context.get_admin_context(show_deleted=True)
self.db_queue = get_scrub_queue()
self.pool = eventlet.greenpool.GreenPool(CONF.scrub_pool_size)
@ -444,9 +339,10 @@ class Scrubber(object):
success = False
if success:
image = self.registry.get_image(image_id)
image = db_api.get_api().image_get(self.admin_context, image_id)
if image['status'] == 'pending_delete':
self.registry.update_image(image_id, {'status': 'deleted'})
db_api.get_api().image_update(self.admin_context, image_id,
{'status': 'deleted'})
LOG.info(_LI("Image %s has been scrubbed successfully"), image_id)
else:
LOG.warn(_LW("One or more image locations couldn't be scrubbed "
@ -454,8 +350,6 @@ class Scrubber(object):
" status") % image_id)
def _delete_image_location_from_backend(self, image_id, loc_id, uri):
if CONF.metadata_encryption_key:
uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri)
try:
LOG.debug("Scrubbing image %s from a location.", image_id)
try:

30
glance/tests/unit/test_scrubber.py
View File

@ -24,6 +24,7 @@ from oslo_config import cfg
from six.moves import range
from glance.common import exception
from glance.db.sqlalchemy import api as db_api
from glance import scrubber
from glance.tests import utils as test_utils
@ -51,9 +52,6 @@ class TestScrubber(test_utils.BaseTestCase):
uri = 'file://some/path/%s' % uuid.uuid4()
id = 'helloworldid'
scrub = scrubber.Scrubber(glance_store)
scrub.registry = self.mox.CreateMockAnything()
scrub.registry.get_image(id).AndReturn({'status': 'pending_delete'})
scrub.registry.update_image(id, {'status': 'deleted'})
self.mox.StubOutWithMock(glance_store, "delete_from_backend")
glance_store.delete_from_backend(
uri,
@ -62,21 +60,20 @@ class TestScrubber(test_utils.BaseTestCase):
scrub._scrub_image(id, [(id, '-', uri)])
self.mox.VerifyAll()
def test_store_delete_successful(self):
@mock.patch.object(db_api, "image_get")
def test_store_delete_successful(self, mock_image_get):
uri = 'file://some/path/%s' % uuid.uuid4()
id = 'helloworldid'
scrub = scrubber.Scrubber(glance_store)
scrub.registry = self.mox.CreateMockAnything()
scrub.registry.get_image(id).AndReturn({'status': 'pending_delete'})
scrub.registry.update_image(id, {'status': 'deleted'})
self.mox.StubOutWithMock(glance_store, "delete_from_backend")
glance_store.delete_from_backend(uri, mox.IgnoreArg()).AndReturn('')
self.mox.ReplayAll()
scrub._scrub_image(id, [(id, '-', uri)])
self.mox.VerifyAll()
def test_store_delete_store_exceptions(self):
@mock.patch.object(db_api, "image_get")
def test_store_delete_store_exceptions(self, mock_image_get):
# While scrubbing image data, all store exceptions, other than
# NotFound, cause image scrubbing to fail. Essentially, no attempt
# would be made to update the status of image.
@ -86,7 +83,6 @@ class TestScrubber(test_utils.BaseTestCase):
ex = glance_store.GlanceStoreException()
scrub = scrubber.Scrubber(glance_store)
scrub.registry = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(glance_store, "delete_from_backend")
glance_store.delete_from_backend(
uri,
@ -95,7 +91,8 @@ class TestScrubber(test_utils.BaseTestCase):
scrub._scrub_image(id, [(id, '-', uri)])
self.mox.VerifyAll()
def test_store_delete_notfound_exception(self):
@mock.patch.object(db_api, "image_get")
def test_store_delete_notfound_exception(self, mock_image_get):
# While scrubbing image data, NotFound exception is ignored and image
# scrubbing succeeds
uri = 'file://some/path/%s' % uuid.uuid4()
@ -103,9 +100,6 @@ class TestScrubber(test_utils.BaseTestCase):
ex = glance_store.NotFound(message='random')
scrub = scrubber.Scrubber(glance_store)
scrub.registry = self.mox.CreateMockAnything()
scrub.registry.get_image(id).AndReturn({'status': 'pending_delete'})
scrub.registry.update_image(id, {'status': 'deleted'})
self.mox.StubOutWithMock(glance_store, "delete_from_backend")
glance_store.delete_from_backend(uri, mox.IgnoreArg()).AndRaise(ex)
self.mox.ReplayAll()
@ -141,11 +135,12 @@ class TestScrubDBQueue(test_utils.BaseTestCase):
image_pager = ImagePager(images)
def make_get_images_detailed(pager):
def mock_get_images_detailed(filters, marker=None):
def mock_get_images_detailed(ctx, filters, marker=None,
limit=None):
return pager()
return mock_get_images_detailed
with patch.object(scrub_queue.registry, 'get_images_detailed') as (
with patch.object(db_api, 'image_get_all') as (
_mock_get_images_detailed):
_mock_get_images_detailed.side_effect = (
make_get_images_detailed(image_pager))
@ -159,11 +154,12 @@ class TestScrubDBQueue(test_utils.BaseTestCase):
image_pager = ImagePager(images, page_size=4)
def make_get_images_detailed(pager):
def mock_get_images_detailed(filters, marker=None):
def mock_get_images_detailed(ctx, filters, marker=None,
limit=None):
return pager()
return mock_get_images_detailed
with patch.object(scrub_queue.registry, 'get_images_detailed') as (
with patch.object(db_api, 'image_get_all') as (
_mock_get_images_detailed):
_mock_get_images_detailed.side_effect = (
make_get_images_detailed(image_pager))

8
releasenotes/notes/scrubber-refactor-73ddbd61ebbf1e86.yaml Normal file
View File

@ -0,0 +1,8 @@
others:
- |
The ``glance-scrubber`` utility, which is used to perfom offline deletion
of images when the Glance ``delayed_delete`` option is enabled, has been
refactored so that it no longer uses the Glance Registry API (and hence no
longer has a dependency on the Registry v1 Client). Configuration options
associated with connecting to the Glance registry are no longer required,
and operators may remove them from the glance-scrubber.conf file.