Refactor scrubber functional test
Refactor scrubber functional test to remove the dependence on Glance V1 and Glance registry. Change-Id: I040b0b8b35d9780206652aedf178cb38481c2020
This commit is contained in:
wangxiyuan
parent
67d35f6166
commit
9e0469743c
@ -565,14 +565,10 @@ log_file = %(log_file)s
|
||||
daemon = %(daemon)s
|
||||
wakeup_time = 2
|
||||
scrub_time = %(scrub_time)s
|
||||
registry_host = %(registry_host)s
|
||||
registry_port = %(registry_port)s
|
||||
metadata_encryption_key = %(metadata_encryption_key)s
|
||||
lock_path = %(lock_path)s
|
||||
sql_connection = %(sql_connection)s
|
||||
sql_idle_timeout = 3600
|
||||
send_identity_headers = %(send_identity_headers)s
|
||||
admin_role = %(admin_role)s
|
||||
[glance_store]
|
||||
filesystem_store_datadir=%(image_dir)s
|
||||
[oslo_policy]
|
||||
|
||||
@ -18,38 +18,59 @@ import sys
|
||||
import time
|
||||
|
||||
import httplib2
|
||||
from oslo_config import cfg
|
||||
from oslo_serialization import jsonutils
|
||||
from oslo_utils import units
|
||||
from six.moves import http_client
|
||||
# NOTE(jokke): simplified transition to py3, behaves like py2 xrange
|
||||
from six.moves import range
|
||||
|
||||
from glance import context
|
||||
import glance.db as db_api
|
||||
from glance.tests import functional
|
||||
from glance.tests.utils import execute
|
||||
|
||||
|
||||
TEST_IMAGE_DATA = '*' * 5 * units.Ki
|
||||
TEST_IMAGE_META = {
|
||||
'name': 'test_image',
|
||||
'is_public': False,
|
||||
'disk_format': 'raw',
|
||||
'container_format': 'ovf',
|
||||
}
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class TestScrubber(functional.FunctionalTest):
|
||||
|
||||
"""Test that delayed_delete works and the scrubber deletes"""
|
||||
|
||||
def _send_http_request(self, path, method, body=None):
|
||||
def setUp(self):
|
||||
super(TestScrubber, self).setUp()
|
||||
self.admin_context = context.get_admin_context(show_deleted=True)
|
||||
CONF.set_override('sql_connection', self.api_server.sql_connection)
|
||||
|
||||
def _send_create_image_http_request(self, path, body=None):
|
||||
headers = {
|
||||
'x-image-meta-name': 'test_image',
|
||||
'x-image-meta-is_public': 'true',
|
||||
'x-image-meta-disk_format': 'raw',
|
||||
'x-image-meta-container_format': 'ovf',
|
||||
'content-type': 'application/octet-stream'
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
return httplib2.Http().request(path, method, body, headers)
|
||||
body = body or {'container_format': 'ovf',
|
||||
'disk_format': 'raw',
|
||||
'name': 'test_image',
|
||||
'visibility': 'public'}
|
||||
body = jsonutils.dumps(body)
|
||||
return httplib2.Http().request(path, 'POST', body, headers)
|
||||
|
||||
def _send_upload_image_http_request(self, path, body=None):
|
||||
headers = {
|
||||
"Content-Type": "application/octet-stream"
|
||||
}
|
||||
return httplib2.Http().request(path, 'PUT', body, headers)
|
||||
|
||||
def _send_http_request(self, path, method):
|
||||
headers = {
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
return httplib2.Http().request(path, method, None, headers)
|
||||
|
||||
def _get_pending_delete_image(self, image_id):
|
||||
# In Glance V2, there is no way to get the 'pending_delete' image from
|
||||
# API. So we get the image from db here for testing.
|
||||
# Clean the session cache first to avoid connecting to the old db data.
|
||||
db_api.get_api()._FACADE = None
|
||||
image = db_api.get_api().image_get(self.admin_context, image_id)
|
||||
return image
|
||||
|
||||
def test_delayed_delete(self):
|
||||
"""
|
||||
@ -59,70 +80,29 @@ class TestScrubber(functional.FunctionalTest):
|
||||
self.cleanup()
|
||||
self.start_servers(delayed_delete=True, daemon=True,
|
||||
metadata_encryption_key='')
|
||||
path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
|
||||
response, content = self._send_http_request(path, 'POST', body='XXX')
|
||||
path = "http://%s:%d/v2/images" % ("127.0.0.1", self.api_port)
|
||||
response, content = self._send_create_image_http_request(path)
|
||||
self.assertEqual(http_client.CREATED, response.status)
|
||||
image = jsonutils.loads(content)['image']
|
||||
image = jsonutils.loads(content)
|
||||
self.assertEqual('queued', image['status'])
|
||||
|
||||
file_path = "%s/%s/file" % (path, image['id'])
|
||||
response, content = self._send_upload_image_http_request(file_path,
|
||||
body='XXX')
|
||||
self.assertEqual(http_client.NO_CONTENT, response.status)
|
||||
|
||||
path = "%s/%s" % (path, image['id'])
|
||||
response, content = self._send_http_request(path, 'GET')
|
||||
image = jsonutils.loads(content)
|
||||
self.assertEqual('active', image['status'])
|
||||
|
||||
path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
|
||||
image['id'])
|
||||
response, content = self._send_http_request(path, 'DELETE')
|
||||
self.assertEqual(http_client.OK, response.status)
|
||||
response, content = self._send_http_request(path, 'HEAD')
|
||||
self.assertEqual(http_client.OK, response.status)
|
||||
self.assertEqual('pending_delete', response['x-image-meta-status'])
|
||||
self.assertEqual(http_client.NO_CONTENT, response.status)
|
||||
|
||||
self.wait_for_scrub(path)
|
||||
image = self._get_pending_delete_image(image['id'])
|
||||
self.assertEqual('pending_delete', image['status'])
|
||||
|
||||
self.stop_servers()
|
||||
|
||||
def test_delayed_delete_with_trustedauth_registry(self):
|
||||
"""
|
||||
test that images don't get deleted immediately and that the scrubber
|
||||
scrubs them when registry is operating in trustedauth mode
|
||||
"""
|
||||
self.cleanup()
|
||||
self.api_server.deployment_flavor = 'noauth'
|
||||
self.registry_server.deployment_flavor = 'trusted-auth'
|
||||
self.start_servers(delayed_delete=True, daemon=True,
|
||||
metadata_encryption_key='',
|
||||
send_identity_headers=True)
|
||||
base_headers = {
|
||||
'X-Identity-Status': 'Confirmed',
|
||||
'X-Auth-Token': '932c5c84-02ac-4fe5-a9ba-620af0e2bb96',
|
||||
'X-User-Id': 'f9a41d13-0c13-47e9-bee2-ce4e8bfe958e',
|
||||
'X-Tenant-Id': 'deae8923-075d-4287-924b-840fb2644874',
|
||||
'X-Roles': 'admin',
|
||||
}
|
||||
headers = {
|
||||
'x-image-meta-name': 'test_image',
|
||||
'x-image-meta-is_public': 'true',
|
||||
'x-image-meta-disk_format': 'raw',
|
||||
'x-image-meta-container_format': 'ovf',
|
||||
'content-type': 'application/octet-stream',
|
||||
}
|
||||
headers.update(base_headers)
|
||||
path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
|
||||
http = httplib2.Http()
|
||||
response, content = http.request(path, 'POST', body='XXX',
|
||||
headers=headers)
|
||||
self.assertEqual(http_client.CREATED, response.status)
|
||||
image = jsonutils.loads(content)['image']
|
||||
self.assertEqual('active', image['status'])
|
||||
image_id = image['id']
|
||||
|
||||
path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
|
||||
image_id)
|
||||
http = httplib2.Http()
|
||||
response, content = http.request(path, 'DELETE', headers=base_headers)
|
||||
self.assertEqual(http_client.OK, response.status)
|
||||
|
||||
response, content = http.request(path, 'HEAD', headers=base_headers)
|
||||
self.assertEqual(http_client.OK, response.status)
|
||||
self.assertEqual('pending_delete', response['x-image-meta-status'])
|
||||
|
||||
self.wait_for_scrub(path, headers=base_headers)
|
||||
self.wait_for_scrub(image['id'])
|
||||
|
||||
self.stop_servers()
|
||||
|
||||
@ -134,21 +114,27 @@ class TestScrubber(functional.FunctionalTest):
|
||||
self.cleanup()
|
||||
self.start_servers(delayed_delete=True, daemon=False,
|
||||
metadata_encryption_key='')
|
||||
|
||||
path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
|
||||
response, content = self._send_http_request(path, 'POST', body='XXX')
|
||||
path = "http://%s:%d/v2/images" % ("127.0.0.1", self.api_port)
|
||||
response, content = self._send_create_image_http_request(path)
|
||||
self.assertEqual(http_client.CREATED, response.status)
|
||||
image = jsonutils.loads(content)['image']
|
||||
image = jsonutils.loads(content)
|
||||
self.assertEqual('queued', image['status'])
|
||||
|
||||
file_path = "%s/%s/file" % (path, image['id'])
|
||||
response, content = self._send_upload_image_http_request(file_path,
|
||||
body='XXX')
|
||||
self.assertEqual(http_client.NO_CONTENT, response.status)
|
||||
|
||||
path = "%s/%s" % (path, image['id'])
|
||||
response, content = self._send_http_request(path, 'GET')
|
||||
image = jsonutils.loads(content)
|
||||
self.assertEqual('active', image['status'])
|
||||
|
||||
path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
|
||||
image['id'])
|
||||
response, content = self._send_http_request(path, 'DELETE')
|
||||
self.assertEqual(http_client.OK, response.status)
|
||||
self.assertEqual(http_client.NO_CONTENT, response.status)
|
||||
|
||||
response, content = self._send_http_request(path, 'HEAD')
|
||||
self.assertEqual(http_client.OK, response.status)
|
||||
self.assertEqual('pending_delete', response['x-image-meta-status'])
|
||||
image = self._get_pending_delete_image(image['id'])
|
||||
self.assertEqual('pending_delete', image['status'])
|
||||
|
||||
# wait for the scrub time on the image to pass
|
||||
time.sleep(self.api_server.scrub_time)
|
||||
@ -160,66 +146,7 @@ class TestScrubber(functional.FunctionalTest):
|
||||
exitcode, out, err = execute(cmd, raise_error=False)
|
||||
self.assertEqual(0, exitcode)
|
||||
|
||||
self.wait_for_scrub(path)
|
||||
|
||||
self.stop_servers()
|
||||
|
||||
def test_scrubber_app_with_trustedauth_registry(self):
|
||||
"""
|
||||
test that the glance-scrubber script runs successfully when not in
|
||||
daemon mode and with a registry that operates in trustedauth mode
|
||||
"""
|
||||
self.cleanup()
|
||||
self.api_server.deployment_flavor = 'noauth'
|
||||
self.registry_server.deployment_flavor = 'trusted-auth'
|
||||
self.start_servers(delayed_delete=True, daemon=False,
|
||||
metadata_encryption_key='',
|
||||
send_identity_headers=True)
|
||||
base_headers = {
|
||||
'X-Identity-Status': 'Confirmed',
|
||||
'X-Auth-Token': '932c5c84-02ac-4fe5-a9ba-620af0e2bb96',
|
||||
'X-User-Id': 'f9a41d13-0c13-47e9-bee2-ce4e8bfe958e',
|
||||
'X-Tenant-Id': 'deae8923-075d-4287-924b-840fb2644874',
|
||||
'X-Roles': 'admin',
|
||||
}
|
||||
headers = {
|
||||
'x-image-meta-name': 'test_image',
|
||||
'x-image-meta-is_public': 'true',
|
||||
'x-image-meta-disk_format': 'raw',
|
||||
'x-image-meta-container_format': 'ovf',
|
||||
'content-type': 'application/octet-stream',
|
||||
}
|
||||
headers.update(base_headers)
|
||||
path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
|
||||
http = httplib2.Http()
|
||||
response, content = http.request(path, 'POST', body='XXX',
|
||||
headers=headers)
|
||||
self.assertEqual(http_client.CREATED, response.status)
|
||||
image = jsonutils.loads(content)['image']
|
||||
self.assertEqual('active', image['status'])
|
||||
image_id = image['id']
|
||||
|
||||
path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
|
||||
image_id)
|
||||
http = httplib2.Http()
|
||||
response, content = http.request(path, 'DELETE', headers=base_headers)
|
||||
self.assertEqual(http_client.OK, response.status)
|
||||
|
||||
response, content = http.request(path, 'HEAD', headers=base_headers)
|
||||
self.assertEqual(http_client.OK, response.status)
|
||||
self.assertEqual('pending_delete', response['x-image-meta-status'])
|
||||
|
||||
# wait for the scrub time on the image to pass
|
||||
time.sleep(self.api_server.scrub_time)
|
||||
|
||||
# scrub images and make sure they get deleted
|
||||
exe_cmd = "%s -m glance.cmd.scrubber" % sys.executable
|
||||
cmd = ("%s --config-file %s" %
|
||||
(exe_cmd, self.scrubber_daemon.conf_file_name))
|
||||
exitcode, out, err = execute(cmd, raise_error=False)
|
||||
self.assertEqual(0, exitcode)
|
||||
|
||||
self.wait_for_scrub(path, headers=base_headers)
|
||||
self.wait_for_scrub(image['id'])
|
||||
|
||||
self.stop_servers()
|
||||
|
||||
@ -239,22 +166,27 @@ class TestScrubber(functional.FunctionalTest):
|
||||
self.assertEqual(self.api_server.default_store, 'file')
|
||||
|
||||
# add an image
|
||||
path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
|
||||
response, content = self._send_http_request(path, 'POST', body='XXX')
|
||||
path = "http://%s:%d/v2/images" % ("127.0.0.1", self.api_port)
|
||||
response, content = self._send_create_image_http_request(path)
|
||||
self.assertEqual(http_client.CREATED, response.status)
|
||||
image = jsonutils.loads(content)['image']
|
||||
image = jsonutils.loads(content)
|
||||
self.assertEqual('queued', image['status'])
|
||||
|
||||
file_path = "%s/%s/file" % (path, image['id'])
|
||||
response, content = self._send_upload_image_http_request(file_path,
|
||||
body='XXX')
|
||||
self.assertEqual(http_client.NO_CONTENT, response.status)
|
||||
|
||||
path = "%s/%s" % (path, image['id'])
|
||||
response, content = self._send_http_request(path, 'GET')
|
||||
image = jsonutils.loads(content)
|
||||
self.assertEqual('active', image['status'])
|
||||
|
||||
# delete the image
|
||||
path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
|
||||
image['id'])
|
||||
response, content = self._send_http_request(path, 'DELETE')
|
||||
self.assertEqual(http_client.OK, response.status)
|
||||
|
||||
# ensure the image is marked pending delete
|
||||
response, content = self._send_http_request(path, 'HEAD')
|
||||
self.assertEqual(http_client.OK, response.status)
|
||||
self.assertEqual('pending_delete', response['x-image-meta-status'])
|
||||
self.assertEqual(http_client.NO_CONTENT, response.status)
|
||||
# ensure the image is marked pending delete.
|
||||
image = self._get_pending_delete_image(image['id'])
|
||||
self.assertEqual('pending_delete', image['status'])
|
||||
|
||||
# Remove the file from the backend.
|
||||
file_path = os.path.join(self.api_server.image_dir, image['id'])
|
||||
@ -270,7 +202,7 @@ class TestScrubber(functional.FunctionalTest):
|
||||
exitcode, out, err = execute(cmd, raise_error=False)
|
||||
self.assertEqual(0, exitcode)
|
||||
|
||||
self.wait_for_scrub(path)
|
||||
self.wait_for_scrub(image['id'])
|
||||
|
||||
self.stop_servers()
|
||||
|
||||
@ -285,7 +217,7 @@ class TestScrubber(functional.FunctionalTest):
|
||||
# Don't start the registry server to cause intended failure
|
||||
# Don't start the api server to save time
|
||||
exitcode, out, err = self.scrubber_daemon.start(
|
||||
delayed_delete=True, daemon=False, registry_port=28890)
|
||||
delayed_delete=True, daemon=False)
|
||||
self.assertEqual(0, exitcode,
|
||||
"Failed to spin up the Scrubber daemon. "
|
||||
"Got: %s" % err)
|
||||
@ -301,21 +233,19 @@ class TestScrubber(functional.FunctionalTest):
|
||||
|
||||
self.stop_server(self.scrubber_daemon)
|
||||
|
||||
def wait_for_scrub(self, path, headers=None):
|
||||
def wait_for_scrub(self, image_id):
|
||||
"""
|
||||
NOTE(jkoelker) The build servers sometimes take longer than 15 seconds
|
||||
to scrub. Give it up to 5 min, checking checking every 15 seconds.
|
||||
When/if it flips to deleted, bail immediately.
|
||||
"""
|
||||
http = httplib2.Http()
|
||||
wait_for = 300 # seconds
|
||||
check_every = 15 # seconds
|
||||
for _ in range(wait_for // check_every):
|
||||
time.sleep(check_every)
|
||||
|
||||
response, content = http.request(path, 'HEAD', headers=headers)
|
||||
if (response['x-image-meta-status'] == 'deleted' and
|
||||
response['x-image-meta-deleted'] == 'True'):
|
||||
image = db_api.get_api().image_get(self.admin_context, image_id)
|
||||
if (image['status'] == 'deleted' and
|
||||
image['deleted'] == True):
|
||||
break
|
||||
else:
|
||||
continue
|
||||
|
||||
Loading…
Reference in New Issue
Block a user