a3f1a9a60b
Deprecates log_format in favor of the formatting options used by the context aware formatter, such as logging_default_format_string. Changes the default logging formatter back to a colorful and context aware formatter. The default was inadvertently changed in the great oslo sync at the start of Havana, this returns it to what was in at the end of Grizzly. Adds support for lazy instantiation of the logger. Bug 1167081 Bug 1165211 Change-Id: Ibe41afc6e9d7b432453785819821b3c8f0078613
972 lines
40 KiB
Python
972 lines
40 KiB
Python
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
|
|
|
# Copyright 2012 Michael Still and Canonical Inc
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
|
|
import contextlib
|
|
import cStringIO
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import time
|
|
|
|
from oslo.config import cfg
|
|
|
|
from nova.compute import vm_states
|
|
from nova import conductor
|
|
from nova import db
|
|
from nova.openstack.common import importutils
|
|
from nova.openstack.common import log as logging
|
|
from nova import test
|
|
from nova import utils
|
|
from nova.virt.libvirt import imagecache
|
|
from nova.virt.libvirt import utils as virtutils
|
|
|
|
CONF = cfg.CONF
|
|
CONF.import_opt('compute_manager', 'nova.service')
|
|
CONF.import_opt('host', 'nova.netconf')
|
|
|
|
|
|
class ImageCacheManagerTestCase(test.TestCase):
|
|
|
|
def setUp(self):
|
|
super(ImageCacheManagerTestCase, self).setUp()
|
|
self.stock_instance_names = set(['instance-00000001',
|
|
'instance-00000002',
|
|
'instance-00000003',
|
|
'banana-42-hamster'])
|
|
|
|
def test_read_stored_checksum_missing(self):
|
|
self.stubs.Set(os.path, 'exists', lambda x: False)
|
|
csum = imagecache.read_stored_checksum('/tmp/foo', timestamped=False)
|
|
self.assertEquals(csum, None)
|
|
|
|
def test_read_stored_checksum(self):
|
|
with utils.tempdir() as tmpdir:
|
|
self.flags(instances_path=tmpdir)
|
|
self.flags(image_info_filename_pattern=('$instances_path/'
|
|
'%(image)s.info'))
|
|
|
|
csum_input = '{"sha1": "fdghkfhkgjjksfdgjksjkghsdf"}\n'
|
|
fname = os.path.join(tmpdir, 'aaa')
|
|
info_fname = imagecache.get_info_filename(fname)
|
|
f = open(info_fname, 'w')
|
|
f.write(csum_input)
|
|
f.close()
|
|
|
|
csum_output = imagecache.read_stored_checksum(fname,
|
|
timestamped=False)
|
|
self.assertEquals(csum_input.rstrip(),
|
|
'{"sha1": "%s"}' % csum_output)
|
|
|
|
def test_read_stored_checksum_legacy_essex(self):
|
|
with utils.tempdir() as tmpdir:
|
|
self.flags(instances_path=tmpdir)
|
|
self.flags(image_info_filename_pattern=('$instances_path/'
|
|
'%(image)s.info'))
|
|
|
|
fname = os.path.join(tmpdir, 'aaa')
|
|
old_fname = fname + '.sha1'
|
|
f = open(old_fname, 'w')
|
|
f.write('fdghkfhkgjjksfdgjksjkghsdf')
|
|
f.close()
|
|
|
|
csum_output = imagecache.read_stored_checksum(fname,
|
|
timestamped=False)
|
|
self.assertEquals(csum_output, 'fdghkfhkgjjksfdgjksjkghsdf')
|
|
self.assertFalse(os.path.exists(old_fname))
|
|
info_fname = imagecache.get_info_filename(fname)
|
|
self.assertTrue(os.path.exists(info_fname))
|
|
|
|
def test_list_base_images(self):
|
|
listing = ['00000001',
|
|
'ephemeral_0_20_None',
|
|
'17d1b00b81642842e514494a78e804e9a511637c_5368709120.info',
|
|
'00000004']
|
|
images = ['e97222e91fc4241f49a7f520d1dcf446751129b3_sm',
|
|
'e09c675c2d1cfac32dae3c2d83689c8c94bc693b_sm',
|
|
'e97222e91fc4241f49a7f520d1dcf446751129b3',
|
|
'17d1b00b81642842e514494a78e804e9a511637c',
|
|
'17d1b00b81642842e514494a78e804e9a511637c_5368709120',
|
|
'17d1b00b81642842e514494a78e804e9a511637c_10737418240']
|
|
listing.extend(images)
|
|
|
|
self.stubs.Set(os, 'listdir', lambda x: listing)
|
|
self.stubs.Set(os.path, 'isfile', lambda x: True)
|
|
|
|
base_dir = '/var/lib/nova/instances/_base'
|
|
self.flags(instances_path='/var/lib/nova/instances')
|
|
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager._list_base_images(base_dir)
|
|
|
|
sanitized = []
|
|
for ent in image_cache_manager.unexplained_images:
|
|
sanitized.append(ent.replace(base_dir + '/', ''))
|
|
|
|
sanitized = sanitized.sort()
|
|
images = images.sort()
|
|
self.assertEquals(sanitized, images)
|
|
|
|
expected = os.path.join(base_dir,
|
|
'e97222e91fc4241f49a7f520d1dcf446751129b3')
|
|
self.assertTrue(expected in image_cache_manager.unexplained_images)
|
|
|
|
expected = os.path.join(base_dir,
|
|
'17d1b00b81642842e514494a78e804e9a511637c_'
|
|
'10737418240')
|
|
self.assertTrue(expected in image_cache_manager.unexplained_images)
|
|
|
|
unexpected = os.path.join(base_dir, '00000004')
|
|
self.assertFalse(unexpected in image_cache_manager.unexplained_images)
|
|
|
|
for ent in image_cache_manager.unexplained_images:
|
|
self.assertTrue(ent.startswith(base_dir))
|
|
|
|
self.assertEquals(len(image_cache_manager.originals), 2)
|
|
|
|
expected = os.path.join(base_dir,
|
|
'17d1b00b81642842e514494a78e804e9a511637c')
|
|
self.assertTrue(expected in image_cache_manager.originals)
|
|
|
|
unexpected = os.path.join(base_dir,
|
|
'17d1b00b81642842e514494a78e804e9a511637c_'
|
|
'10737418240')
|
|
self.assertFalse(unexpected in image_cache_manager.originals)
|
|
|
|
def test_list_running_instances(self):
|
|
all_instances = [{'image_ref': '1',
|
|
'host': CONF.host,
|
|
'name': 'inst-1',
|
|
'uuid': '123',
|
|
'vm_state': '',
|
|
'task_state': ''},
|
|
{'image_ref': '2',
|
|
'host': CONF.host,
|
|
'name': 'inst-2',
|
|
'uuid': '456',
|
|
'vm_state': '',
|
|
'task_state': ''},
|
|
{'image_ref': '2',
|
|
'kernel_id': '21',
|
|
'ramdisk_id': '22',
|
|
'host': 'remotehost',
|
|
'name': 'inst-3',
|
|
'uuid': '789',
|
|
'vm_state': '',
|
|
'task_state': ''}]
|
|
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
|
|
# The argument here should be a context, but it's mocked out
|
|
image_cache_manager._list_running_instances(None, all_instances)
|
|
|
|
self.assertEqual(len(image_cache_manager.used_images), 4)
|
|
self.assertTrue(image_cache_manager.used_images['1'] ==
|
|
(1, 0, ['inst-1']))
|
|
self.assertTrue(image_cache_manager.used_images['2'] ==
|
|
(1, 1, ['inst-2', 'inst-3']))
|
|
self.assertTrue(image_cache_manager.used_images['21'] ==
|
|
(0, 1, ['inst-3']))
|
|
self.assertTrue(image_cache_manager.used_images['22'] ==
|
|
(0, 1, ['inst-3']))
|
|
|
|
self.assertTrue('inst-1' in image_cache_manager.instance_names)
|
|
self.assertTrue('123' in image_cache_manager.instance_names)
|
|
|
|
self.assertEqual(len(image_cache_manager.image_popularity), 4)
|
|
self.assertEqual(image_cache_manager.image_popularity['1'], 1)
|
|
self.assertEqual(image_cache_manager.image_popularity['2'], 2)
|
|
self.assertEqual(image_cache_manager.image_popularity['21'], 1)
|
|
self.assertEqual(image_cache_manager.image_popularity['22'], 1)
|
|
|
|
def test_list_resizing_instances(self):
|
|
all_instances = [{'image_ref': '1',
|
|
'host': CONF.host,
|
|
'name': 'inst-1',
|
|
'uuid': '123',
|
|
'vm_state': vm_states.RESIZED,
|
|
'task_state': None}]
|
|
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager._list_running_instances(None, all_instances)
|
|
|
|
self.assertEqual(len(image_cache_manager.used_images), 1)
|
|
self.assertTrue(image_cache_manager.used_images['1'] ==
|
|
(1, 0, ['inst-1']))
|
|
self.assertTrue(image_cache_manager.instance_names ==
|
|
set(['inst-1', '123', 'inst-1_resize', '123_resize']))
|
|
|
|
self.assertEqual(len(image_cache_manager.image_popularity), 1)
|
|
self.assertEqual(image_cache_manager.image_popularity['1'], 1)
|
|
|
|
def test_list_backing_images_small(self):
|
|
self.stubs.Set(os, 'listdir',
|
|
lambda x: ['_base', 'instance-00000001',
|
|
'instance-00000002', 'instance-00000003'])
|
|
self.stubs.Set(os.path, 'exists',
|
|
lambda x: x.find('instance-') != -1)
|
|
self.stubs.Set(virtutils, 'get_disk_backing_file',
|
|
lambda x: 'e97222e91fc4241f49a7f520d1dcf446751129b3_sm')
|
|
|
|
found = os.path.join(CONF.instances_path, CONF.base_dir_name,
|
|
'e97222e91fc4241f49a7f520d1dcf446751129b3_sm')
|
|
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager.unexplained_images = [found]
|
|
image_cache_manager.instance_names = self.stock_instance_names
|
|
|
|
inuse_images = image_cache_manager._list_backing_images()
|
|
|
|
self.assertEquals(inuse_images, [found])
|
|
self.assertEquals(len(image_cache_manager.unexplained_images), 0)
|
|
|
|
def test_list_backing_images_resized(self):
|
|
self.stubs.Set(os, 'listdir',
|
|
lambda x: ['_base', 'instance-00000001',
|
|
'instance-00000002', 'instance-00000003'])
|
|
self.stubs.Set(os.path, 'exists',
|
|
lambda x: x.find('instance-') != -1)
|
|
self.stubs.Set(virtutils, 'get_disk_backing_file',
|
|
lambda x: ('e97222e91fc4241f49a7f520d1dcf446751129b3_'
|
|
'10737418240'))
|
|
|
|
found = os.path.join(CONF.instances_path, CONF.base_dir_name,
|
|
'e97222e91fc4241f49a7f520d1dcf446751129b3_'
|
|
'10737418240')
|
|
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager.unexplained_images = [found]
|
|
image_cache_manager.instance_names = self.stock_instance_names
|
|
|
|
inuse_images = image_cache_manager._list_backing_images()
|
|
|
|
self.assertEquals(inuse_images, [found])
|
|
self.assertEquals(len(image_cache_manager.unexplained_images), 0)
|
|
|
|
def test_list_backing_images_instancename(self):
|
|
self.stubs.Set(os, 'listdir',
|
|
lambda x: ['_base', 'banana-42-hamster'])
|
|
self.stubs.Set(os.path, 'exists',
|
|
lambda x: x.find('banana-42-hamster') != -1)
|
|
self.stubs.Set(virtutils, 'get_disk_backing_file',
|
|
lambda x: 'e97222e91fc4241f49a7f520d1dcf446751129b3_sm')
|
|
|
|
found = os.path.join(CONF.instances_path, CONF.base_dir_name,
|
|
'e97222e91fc4241f49a7f520d1dcf446751129b3_sm')
|
|
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager.unexplained_images = [found]
|
|
image_cache_manager.instance_names = self.stock_instance_names
|
|
|
|
inuse_images = image_cache_manager._list_backing_images()
|
|
|
|
self.assertEquals(inuse_images, [found])
|
|
self.assertEquals(len(image_cache_manager.unexplained_images), 0)
|
|
|
|
def test_find_base_file_nothing(self):
|
|
self.stubs.Set(os.path, 'exists', lambda x: False)
|
|
|
|
base_dir = '/var/lib/nova/instances/_base'
|
|
fingerprint = '549867354867'
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
res = list(image_cache_manager._find_base_file(base_dir, fingerprint))
|
|
|
|
self.assertEqual(0, len(res))
|
|
|
|
def test_find_base_file_small(self):
|
|
fingerprint = '968dd6cc49e01aaa044ed11c0cce733e0fa44a6a'
|
|
self.stubs.Set(os.path, 'exists',
|
|
lambda x: x.endswith('%s_sm' % fingerprint))
|
|
|
|
base_dir = '/var/lib/nova/instances/_base'
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
res = list(image_cache_manager._find_base_file(base_dir, fingerprint))
|
|
|
|
base_file = os.path.join(base_dir, fingerprint + '_sm')
|
|
self.assertTrue(res == [(base_file, True, False)])
|
|
|
|
def test_find_base_file_resized(self):
|
|
fingerprint = '968dd6cc49e01aaa044ed11c0cce733e0fa44a6a'
|
|
listing = ['00000001',
|
|
'ephemeral_0_20_None',
|
|
'968dd6cc49e01aaa044ed11c0cce733e0fa44a6a_10737418240',
|
|
'00000004']
|
|
|
|
self.stubs.Set(os, 'listdir', lambda x: listing)
|
|
self.stubs.Set(os.path, 'exists',
|
|
lambda x: x.endswith('%s_10737418240' % fingerprint))
|
|
self.stubs.Set(os.path, 'isfile', lambda x: True)
|
|
|
|
base_dir = '/var/lib/nova/instances/_base'
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager._list_base_images(base_dir)
|
|
res = list(image_cache_manager._find_base_file(base_dir, fingerprint))
|
|
|
|
base_file = os.path.join(base_dir, fingerprint + '_10737418240')
|
|
self.assertTrue(res == [(base_file, False, True)])
|
|
|
|
def test_find_base_file_all(self):
|
|
fingerprint = '968dd6cc49e01aaa044ed11c0cce733e0fa44a6a'
|
|
listing = ['00000001',
|
|
'ephemeral_0_20_None',
|
|
'968dd6cc49e01aaa044ed11c0cce733e0fa44a6a_sm',
|
|
'968dd6cc49e01aaa044ed11c0cce733e0fa44a6a_10737418240',
|
|
'00000004']
|
|
|
|
self.stubs.Set(os, 'listdir', lambda x: listing)
|
|
self.stubs.Set(os.path, 'exists', lambda x: True)
|
|
self.stubs.Set(os.path, 'isfile', lambda x: True)
|
|
|
|
base_dir = '/var/lib/nova/instances/_base'
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager._list_base_images(base_dir)
|
|
res = list(image_cache_manager._find_base_file(base_dir, fingerprint))
|
|
|
|
base_file1 = os.path.join(base_dir, fingerprint)
|
|
base_file2 = os.path.join(base_dir, fingerprint + '_sm')
|
|
base_file3 = os.path.join(base_dir, fingerprint + '_10737418240')
|
|
self.assertTrue(res == [(base_file1, False, False),
|
|
(base_file2, True, False),
|
|
(base_file3, False, True)])
|
|
|
|
@contextlib.contextmanager
|
|
def _intercept_log_messages(self):
|
|
try:
|
|
mylog = logging.getLogger('nova')
|
|
stream = cStringIO.StringIO()
|
|
handler = logging.logging.StreamHandler(stream)
|
|
handler.setFormatter(logging.ContextFormatter())
|
|
mylog.logger.addHandler(handler)
|
|
yield stream
|
|
finally:
|
|
mylog.logger.removeHandler(handler)
|
|
|
|
def _make_checksum(self, tmpdir):
|
|
testdata = ('OpenStack Software delivers a massively scalable cloud '
|
|
'operating system.')
|
|
|
|
fname = os.path.join(tmpdir, 'aaa')
|
|
info_fname = imagecache.get_info_filename(fname)
|
|
|
|
with open(fname, 'w') as f:
|
|
f.write(testdata)
|
|
|
|
return fname, info_fname, testdata
|
|
|
|
def test_verify_checksum(self):
|
|
img = {'container_format': 'ami', 'id': '42'}
|
|
|
|
self.flags(checksum_base_images=True)
|
|
|
|
with self._intercept_log_messages() as stream:
|
|
with utils.tempdir() as tmpdir:
|
|
self.flags(instances_path=tmpdir)
|
|
self.flags(image_info_filename_pattern=('$instances_path/'
|
|
'%(image)s.info'))
|
|
fname, info_fname, testdata = self._make_checksum(tmpdir)
|
|
|
|
# Checksum is valid
|
|
f = open(info_fname, 'w')
|
|
csum = hashlib.sha1()
|
|
csum.update(testdata)
|
|
f.write('{"sha1": "%s"}\n' % csum.hexdigest())
|
|
f.close()
|
|
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
res = image_cache_manager._verify_checksum(img, fname)
|
|
self.assertTrue(res)
|
|
|
|
def test_verify_checksum_disabled(self):
|
|
img = {'container_format': 'ami', 'id': '42'}
|
|
|
|
self.flags(checksum_base_images=False)
|
|
|
|
with self._intercept_log_messages() as stream:
|
|
with utils.tempdir() as tmpdir:
|
|
self.flags(instances_path=tmpdir)
|
|
self.flags(image_info_filename_pattern=('$instances_path/'
|
|
'%(image)s.info'))
|
|
fname, info_fname, testdata = self._make_checksum(tmpdir)
|
|
|
|
# Checksum is valid
|
|
f = open(info_fname, 'w')
|
|
csum = hashlib.sha1()
|
|
csum.update(testdata)
|
|
f.write('{"sha1": "%s"}\n' % csum.hexdigest())
|
|
f.close()
|
|
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
res = image_cache_manager._verify_checksum(img, fname)
|
|
self.assertTrue(res is None)
|
|
|
|
def test_verify_checksum_invalid_json(self):
|
|
img = {'container_format': 'ami', 'id': '42'}
|
|
|
|
self.flags(checksum_base_images=True)
|
|
|
|
with self._intercept_log_messages() as stream:
|
|
with utils.tempdir() as tmpdir:
|
|
self.flags(instances_path=tmpdir)
|
|
self.flags(image_info_filename_pattern=('$instances_path/'
|
|
'%(image)s.info'))
|
|
|
|
fname, info_fname, testdata = self._make_checksum(tmpdir)
|
|
|
|
# Checksum is invalid, and not json
|
|
f = open(info_fname, 'w')
|
|
f.write('banana')
|
|
f.close()
|
|
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
res = image_cache_manager._verify_checksum(
|
|
img, fname, create_if_missing=False)
|
|
self.assertFalse(res)
|
|
log = stream.getvalue()
|
|
|
|
# NOTE(mikal): this is a skip not a fail because the file is
|
|
# present, but is not in valid json format and therefore is
|
|
# skipped.
|
|
self.assertNotEqual(log.find('image verification skipped'), -1)
|
|
|
|
def test_verify_checksum_invalid_repaired(self):
|
|
img = {'container_format': 'ami', 'id': '42'}
|
|
|
|
self.flags(checksum_base_images=True)
|
|
|
|
with utils.tempdir() as tmpdir:
|
|
self.flags(instances_path=tmpdir)
|
|
self.flags(image_info_filename_pattern=('$instances_path/'
|
|
'%(image)s.info'))
|
|
|
|
fname, info_fname, testdata = self._make_checksum(tmpdir)
|
|
|
|
# Checksum is invalid, and not json
|
|
f = open(info_fname, 'w')
|
|
f.write('banana')
|
|
f.close()
|
|
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
res = image_cache_manager._verify_checksum(
|
|
img, fname, create_if_missing=True)
|
|
self.assertTrue(res is None)
|
|
|
|
def test_verify_checksum_invalid(self):
|
|
img = {'container_format': 'ami', 'id': '42'}
|
|
|
|
self.flags(checksum_base_images=True)
|
|
|
|
with self._intercept_log_messages() as stream:
|
|
with utils.tempdir() as tmpdir:
|
|
self.flags(instances_path=tmpdir)
|
|
self.flags(image_info_filename_pattern=('$instances_path/'
|
|
'%(image)s.info'))
|
|
|
|
fname, info_fname, testdata = self._make_checksum(tmpdir)
|
|
|
|
# Checksum is invalid, but is in valid json
|
|
f = open(info_fname, 'w')
|
|
f.write('{"sha1": "banana"}')
|
|
f.close()
|
|
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
res = image_cache_manager._verify_checksum(img, fname)
|
|
self.assertFalse(res)
|
|
log = stream.getvalue()
|
|
|
|
self.assertNotEqual(log.find('image verification failed'), -1)
|
|
|
|
def test_verify_checksum_file_missing(self):
|
|
self.flags(checksum_base_images=True)
|
|
|
|
with utils.tempdir() as tmpdir:
|
|
self.flags(instances_path=tmpdir)
|
|
self.flags(image_info_filename_pattern=('$instances_path/'
|
|
'%(image)s.info'))
|
|
|
|
fname, info_fname, testdata = self._make_checksum(tmpdir)
|
|
|
|
# Checksum file missing
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
res = image_cache_manager._verify_checksum('aaa', fname)
|
|
self.assertEquals(res, None)
|
|
|
|
# Checksum requests for a file with no checksum now have the
|
|
# side effect of creating the checksum
|
|
self.assertTrue(os.path.exists(info_fname))
|
|
|
|
@contextlib.contextmanager
|
|
def _make_base_file(self, checksum=True):
|
|
"""Make a base file for testing."""
|
|
|
|
with utils.tempdir() as tmpdir:
|
|
self.flags(instances_path=tmpdir)
|
|
self.flags(image_info_filename_pattern=('$instances_path/'
|
|
'%(image)s.info'))
|
|
fname = os.path.join(tmpdir, 'aaa')
|
|
|
|
base_file = open(fname, 'w')
|
|
base_file.write('data')
|
|
base_file.close()
|
|
base_file = open(fname, 'r')
|
|
|
|
if checksum:
|
|
imagecache.write_stored_checksum(fname)
|
|
|
|
base_file.close()
|
|
yield fname
|
|
|
|
def test_remove_base_file(self):
|
|
with self._make_base_file() as fname:
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager._remove_base_file(fname)
|
|
info_fname = imagecache.get_info_filename(fname)
|
|
|
|
# Files are initially too new to delete
|
|
self.assertTrue(os.path.exists(fname))
|
|
self.assertTrue(os.path.exists(info_fname))
|
|
|
|
# Old files get cleaned up though
|
|
os.utime(fname, (-1, time.time() - 3601))
|
|
image_cache_manager._remove_base_file(fname)
|
|
|
|
self.assertFalse(os.path.exists(fname))
|
|
self.assertFalse(os.path.exists(info_fname))
|
|
|
|
def test_remove_base_file_original(self):
|
|
with self._make_base_file() as fname:
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager.originals = [fname]
|
|
image_cache_manager._remove_base_file(fname)
|
|
info_fname = imagecache.get_info_filename(fname)
|
|
|
|
# Files are initially too new to delete
|
|
self.assertTrue(os.path.exists(fname))
|
|
self.assertTrue(os.path.exists(info_fname))
|
|
|
|
# This file should stay longer than a resized image
|
|
os.utime(fname, (-1, time.time() - 3601))
|
|
image_cache_manager._remove_base_file(fname)
|
|
|
|
self.assertTrue(os.path.exists(fname))
|
|
self.assertTrue(os.path.exists(info_fname))
|
|
|
|
# Originals don't stay forever though
|
|
os.utime(fname, (-1, time.time() - 3600 * 25))
|
|
image_cache_manager._remove_base_file(fname)
|
|
|
|
self.assertFalse(os.path.exists(fname))
|
|
self.assertFalse(os.path.exists(info_fname))
|
|
|
|
def test_remove_base_file_dne(self):
|
|
# This test is solely to execute the "does not exist" code path. We
|
|
# don't expect the method being tested to do anything in this case.
|
|
with utils.tempdir() as tmpdir:
|
|
self.flags(instances_path=tmpdir)
|
|
self.flags(image_info_filename_pattern=('$instances_path/'
|
|
'%(image)s.info'))
|
|
|
|
fname = os.path.join(tmpdir, 'aaa')
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager._remove_base_file(fname)
|
|
|
|
def test_remove_base_file_oserror(self):
|
|
with self._intercept_log_messages() as stream:
|
|
with utils.tempdir() as tmpdir:
|
|
self.flags(instances_path=tmpdir)
|
|
self.flags(image_info_filename_pattern=('$instances_path/'
|
|
'%(image)s.info'))
|
|
|
|
fname = os.path.join(tmpdir, 'aaa')
|
|
|
|
os.mkdir(fname)
|
|
os.utime(fname, (-1, time.time() - 3601))
|
|
|
|
# This will raise an OSError because of file permissions
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager._remove_base_file(fname)
|
|
|
|
self.assertTrue(os.path.exists(fname))
|
|
self.assertNotEqual(stream.getvalue().find('Failed to remove'),
|
|
-1)
|
|
|
|
def test_handle_base_image_unused(self):
|
|
img = '123'
|
|
|
|
with self._make_base_file() as fname:
|
|
os.utime(fname, (-1, time.time() - 3601))
|
|
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager.unexplained_images = [fname]
|
|
image_cache_manager._handle_base_image(img, fname)
|
|
|
|
self.assertEquals(image_cache_manager.unexplained_images, [])
|
|
self.assertEquals(image_cache_manager.removable_base_files,
|
|
[fname])
|
|
self.assertEquals(image_cache_manager.corrupt_base_files, [])
|
|
|
|
def test_handle_base_image_used(self):
|
|
self.stubs.Set(virtutils, 'chown', lambda x, y: None)
|
|
img = '123'
|
|
|
|
with self._make_base_file() as fname:
|
|
os.utime(fname, (-1, time.time() - 3601))
|
|
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager.unexplained_images = [fname]
|
|
image_cache_manager.used_images = {'123': (1, 0, ['banana-42'])}
|
|
image_cache_manager._handle_base_image(img, fname)
|
|
|
|
self.assertEquals(image_cache_manager.unexplained_images, [])
|
|
self.assertEquals(image_cache_manager.removable_base_files, [])
|
|
self.assertEquals(image_cache_manager.corrupt_base_files, [])
|
|
|
|
def test_handle_base_image_used_remotely(self):
|
|
self.stubs.Set(virtutils, 'chown', lambda x, y: None)
|
|
img = '123'
|
|
|
|
with self._make_base_file() as fname:
|
|
os.utime(fname, (-1, time.time() - 3601))
|
|
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager.unexplained_images = [fname]
|
|
image_cache_manager.used_images = {'123': (0, 1, ['banana-42'])}
|
|
image_cache_manager._handle_base_image(img, fname)
|
|
|
|
self.assertEquals(image_cache_manager.unexplained_images, [])
|
|
self.assertEquals(image_cache_manager.removable_base_files, [])
|
|
self.assertEquals(image_cache_manager.corrupt_base_files, [])
|
|
|
|
def test_handle_base_image_absent(self):
|
|
img = '123'
|
|
|
|
with self._intercept_log_messages() as stream:
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager.used_images = {'123': (1, 0, ['banana-42'])}
|
|
image_cache_manager._handle_base_image(img, None)
|
|
|
|
self.assertEquals(image_cache_manager.unexplained_images, [])
|
|
self.assertEquals(image_cache_manager.removable_base_files, [])
|
|
self.assertEquals(image_cache_manager.corrupt_base_files, [])
|
|
self.assertNotEqual(stream.getvalue().find('an absent base file'),
|
|
-1)
|
|
|
|
def test_handle_base_image_used_missing(self):
|
|
img = '123'
|
|
|
|
with utils.tempdir() as tmpdir:
|
|
self.flags(instances_path=tmpdir)
|
|
self.flags(image_info_filename_pattern=('$instances_path/'
|
|
'%(image)s.info'))
|
|
|
|
fname = os.path.join(tmpdir, 'aaa')
|
|
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager.unexplained_images = [fname]
|
|
image_cache_manager.used_images = {'123': (1, 0, ['banana-42'])}
|
|
image_cache_manager._handle_base_image(img, fname)
|
|
|
|
self.assertEquals(image_cache_manager.unexplained_images, [])
|
|
self.assertEquals(image_cache_manager.removable_base_files, [])
|
|
self.assertEquals(image_cache_manager.corrupt_base_files, [])
|
|
|
|
def test_handle_base_image_checksum_fails(self):
|
|
self.flags(checksum_base_images=True)
|
|
self.stubs.Set(virtutils, 'chown', lambda x, y: None)
|
|
|
|
img = '123'
|
|
|
|
with self._make_base_file() as fname:
|
|
with open(fname, 'w') as f:
|
|
f.write('banana')
|
|
|
|
d = {'sha1': '21323454'}
|
|
with open('%s.info' % fname, 'w') as f:
|
|
f.write(json.dumps(d))
|
|
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager.unexplained_images = [fname]
|
|
image_cache_manager.used_images = {'123': (1, 0, ['banana-42'])}
|
|
image_cache_manager._handle_base_image(img, fname)
|
|
|
|
self.assertEquals(image_cache_manager.unexplained_images, [])
|
|
self.assertEquals(image_cache_manager.removable_base_files, [])
|
|
self.assertEquals(image_cache_manager.corrupt_base_files,
|
|
[fname])
|
|
|
|
def test_verify_base_images(self):
|
|
hashed_1 = '356a192b7913b04c54574d18c28d46e6395428ab'
|
|
hashed_21 = '472b07b9fcf2c2451e8781e944bf5f77cd8457c8'
|
|
hashed_22 = '12c6fc06c99a462375eeb3f43dfd832b08ca9e17'
|
|
hashed_42 = '92cfceb39d57d914ed8b14d0e37643de0797ae56'
|
|
|
|
self.flags(instances_path='/instance_path')
|
|
self.flags(base_dir_name='_base')
|
|
self.flags(remove_unused_base_images=True)
|
|
|
|
base_file_list = ['00000001',
|
|
'ephemeral_0_20_None',
|
|
'e97222e91fc4241f49a7f520d1dcf446751129b3_sm',
|
|
'e09c675c2d1cfac32dae3c2d83689c8c94bc693b_sm',
|
|
hashed_42,
|
|
hashed_1,
|
|
hashed_21,
|
|
hashed_22,
|
|
'%s_5368709120' % hashed_1,
|
|
'%s_10737418240' % hashed_1,
|
|
'00000004']
|
|
|
|
def fq_path(path):
|
|
return os.path.join('/instance_path/_base/', path)
|
|
|
|
# Fake base directory existence
|
|
orig_exists = os.path.exists
|
|
|
|
def exists(path):
|
|
# The python coverage tool got angry with my overly broad mocks
|
|
if not path.startswith('/instance_path'):
|
|
return orig_exists(path)
|
|
|
|
if path in ['/instance_path',
|
|
'/instance_path/_base',
|
|
'/instance_path/instance-1/disk',
|
|
'/instance_path/instance-2/disk',
|
|
'/instance_path/instance-3/disk',
|
|
'/instance_path/_base/%s.info' % hashed_42]:
|
|
return True
|
|
|
|
for p in base_file_list:
|
|
if path == fq_path(p):
|
|
return True
|
|
if path == fq_path(p) + '.info':
|
|
return False
|
|
|
|
if path in ['/instance_path/_base/%s_sm' % i for i in [hashed_1,
|
|
hashed_21,
|
|
hashed_22,
|
|
hashed_42]]:
|
|
return False
|
|
|
|
self.fail('Unexpected path existence check: %s' % path)
|
|
|
|
self.stubs.Set(os.path, 'exists', lambda x: exists(x))
|
|
|
|
self.stubs.Set(virtutils, 'chown', lambda x, y: None)
|
|
|
|
# We need to stub utime as well
|
|
orig_utime = os.utime
|
|
self.stubs.Set(os, 'utime', lambda x, y: None)
|
|
|
|
# Fake up some instances in the instances directory
|
|
orig_listdir = os.listdir
|
|
|
|
def listdir(path):
|
|
# The python coverage tool got angry with my overly broad mocks
|
|
if not path.startswith('/instance_path'):
|
|
return orig_listdir(path)
|
|
|
|
if path == '/instance_path':
|
|
return ['instance-1', 'instance-2', 'instance-3', '_base']
|
|
|
|
if path == '/instance_path/_base':
|
|
return base_file_list
|
|
|
|
self.fail('Unexpected directory listed: %s' % path)
|
|
|
|
self.stubs.Set(os, 'listdir', lambda x: listdir(x))
|
|
|
|
# Fake isfile for these faked images in _base
|
|
orig_isfile = os.path.isfile
|
|
|
|
def isfile(path):
|
|
# The python coverage tool got angry with my overly broad mocks
|
|
if not path.startswith('/instance_path'):
|
|
return orig_isfile(path)
|
|
|
|
for p in base_file_list:
|
|
if path == fq_path(p):
|
|
return True
|
|
|
|
self.fail('Unexpected isfile call: %s' % path)
|
|
|
|
self.stubs.Set(os.path, 'isfile', lambda x: isfile(x))
|
|
|
|
# Fake the database call which lists running instances
|
|
all_instances = [{'image_ref': '1',
|
|
'host': CONF.host,
|
|
'name': 'instance-1',
|
|
'uuid': '123',
|
|
'vm_state': '',
|
|
'task_state': ''},
|
|
{'image_ref': '1',
|
|
'kernel_id': '21',
|
|
'ramdisk_id': '22',
|
|
'host': CONF.host,
|
|
'name': 'instance-2',
|
|
'uuid': '456',
|
|
'vm_state': '',
|
|
'task_state': ''}]
|
|
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
|
|
# Fake the utils call which finds the backing image
|
|
def get_disk_backing_file(path):
|
|
if path in ['/instance_path/instance-1/disk',
|
|
'/instance_path/instance-2/disk']:
|
|
return fq_path('%s_5368709120' % hashed_1)
|
|
self.fail('Unexpected backing file lookup: %s' % path)
|
|
|
|
self.stubs.Set(virtutils, 'get_disk_backing_file',
|
|
lambda x: get_disk_backing_file(x))
|
|
|
|
# Fake out verifying checksums, as that is tested elsewhere
|
|
self.stubs.Set(image_cache_manager, '_verify_checksum',
|
|
lambda x, y: y == hashed_42)
|
|
|
|
# Fake getmtime as well
|
|
orig_getmtime = os.path.getmtime
|
|
|
|
def getmtime(path):
|
|
if not path.startswith('/instance_path'):
|
|
return orig_getmtime(path)
|
|
|
|
return 1000000
|
|
|
|
self.stubs.Set(os.path, 'getmtime', lambda x: getmtime(x))
|
|
|
|
# Make sure we don't accidentally remove a real file
|
|
orig_remove = os.remove
|
|
|
|
def remove(path):
|
|
if not path.startswith('/instance_path'):
|
|
return orig_remove(path)
|
|
|
|
# Don't try to remove fake files
|
|
return
|
|
|
|
self.stubs.Set(os, 'remove', lambda x: remove(x))
|
|
|
|
# And finally we can make the call we're actually testing...
|
|
# The argument here should be a context, but it is mocked out
|
|
image_cache_manager.verify_base_images(None, all_instances)
|
|
|
|
# Verify
|
|
active = [fq_path(hashed_1), fq_path('%s_5368709120' % hashed_1),
|
|
fq_path(hashed_21), fq_path(hashed_22)]
|
|
self.assertEquals(image_cache_manager.active_base_files, active)
|
|
|
|
for rem in [fq_path('e97222e91fc4241f49a7f520d1dcf446751129b3_sm'),
|
|
fq_path('e09c675c2d1cfac32dae3c2d83689c8c94bc693b_sm'),
|
|
fq_path(hashed_42),
|
|
fq_path('%s_10737418240' % hashed_1)]:
|
|
self.assertTrue(rem in image_cache_manager.removable_base_files)
|
|
|
|
# Ensure there are no "corrupt" images as well
|
|
self.assertTrue(len(image_cache_manager.corrupt_base_files), 0)
|
|
|
|
def test_verify_base_images_no_base(self):
|
|
self.flags(instances_path='/tmp/no/such/dir/name/please')
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager.verify_base_images(None, [])
|
|
|
|
def test_is_valid_info_file(self):
|
|
hashed = 'e97222e91fc4241f49a7f520d1dcf446751129b3'
|
|
|
|
self.flags(instances_path='/tmp/no/such/dir/name/please')
|
|
self.flags(image_info_filename_pattern=('$instances_path/_base/'
|
|
'%(image)s.info'))
|
|
base_filename = os.path.join(CONF.instances_path, '_base', hashed)
|
|
|
|
is_valid_info_file = imagecache.is_valid_info_file
|
|
self.assertFalse(is_valid_info_file('banana'))
|
|
self.assertFalse(is_valid_info_file(
|
|
os.path.join(CONF.instances_path, '_base', '00000001')))
|
|
self.assertFalse(is_valid_info_file(base_filename))
|
|
self.assertFalse(is_valid_info_file(base_filename + '.sha1'))
|
|
self.assertTrue(is_valid_info_file(base_filename + '.info'))
|
|
|
|
def test_configured_checksum_path(self):
|
|
with utils.tempdir() as tmpdir:
|
|
self.flags(instances_path=tmpdir)
|
|
self.flags(image_info_filename_pattern=('$instances_path/'
|
|
'%(image)s.info'))
|
|
self.flags(remove_unused_base_images=True)
|
|
|
|
# Ensure there is a base directory
|
|
os.mkdir(os.path.join(tmpdir, '_base'))
|
|
|
|
# Fake the database call which lists running instances
|
|
all_instances = [{'image_ref': '1',
|
|
'host': CONF.host,
|
|
'name': 'instance-1',
|
|
'uuid': '123',
|
|
'vm_state': '',
|
|
'task_state': ''},
|
|
{'image_ref': '1',
|
|
'host': CONF.host,
|
|
'name': 'instance-2',
|
|
'uuid': '456',
|
|
'vm_state': '',
|
|
'task_state': ''}]
|
|
|
|
def touch(filename):
|
|
f = open(filename, 'w')
|
|
f.write('Touched')
|
|
f.close()
|
|
|
|
old = time.time() - (25 * 3600)
|
|
hashed = 'e97222e91fc4241f49a7f520d1dcf446751129b3'
|
|
base_filename = os.path.join(tmpdir, hashed)
|
|
touch(base_filename)
|
|
touch(base_filename + '.info')
|
|
os.utime(base_filename + '.info', (old, old))
|
|
touch(base_filename + '.info')
|
|
os.utime(base_filename + '.info', (old, old))
|
|
|
|
image_cache_manager = imagecache.ImageCacheManager()
|
|
image_cache_manager.verify_base_images(None, all_instances)
|
|
|
|
self.assertTrue(os.path.exists(base_filename))
|
|
self.assertTrue(os.path.exists(base_filename + '.info'))
|
|
|
|
def test_compute_manager(self):
|
|
was = {'called': False}
|
|
|
|
def fake_get_all(context, *args, **kwargs):
|
|
was['called'] = True
|
|
return [{'image_ref': '1',
|
|
'host': CONF.host,
|
|
'name': 'instance-1',
|
|
'uuid': '123',
|
|
'vm_state': '',
|
|
'task_state': ''},
|
|
{'image_ref': '1',
|
|
'host': CONF.host,
|
|
'name': 'instance-2',
|
|
'uuid': '456',
|
|
'vm_state': '',
|
|
'task_state': ''}]
|
|
|
|
with utils.tempdir() as tmpdir:
|
|
self.flags(instances_path=tmpdir)
|
|
|
|
self.stubs.Set(db, 'instance_get_all', fake_get_all)
|
|
compute = importutils.import_object(CONF.compute_manager)
|
|
self.flags(use_local=True, group='conductor')
|
|
compute.conductor_api = conductor.API()
|
|
compute._run_image_cache_manager_pass(None)
|
|
self.assertTrue(was['called'])
|