openstack/nova
Code Issues Proposed changes

Switch to the oslo_utils.fileutils

Browse Source 
fileutils is graduated in the oslo.utils library.
Remove old implementation of the read_cached_file function
in the nova.utils and move from the fileutils the read_cached_file
and delete_cached_file functions to the nova.utils module and
the unit tests for these functions.

Implements: blueprint graduate-fileutils[1]
[1] https://blueprints.launchpad.net/oslo-incubator/+spec/graduate-fileutils

Depends-On: I51ba9076e1fbc16145ee2311f47b7768c16dcb20 (requirements)

Change-Id: I849f1c74ec811dbe82ba6270569a008a49eab465
This commit is contained in:
Sergey Vilgelm 2015-07-08 16:45:36 +03:00
parent 0d696adcea
commit 741f163fc5
26 changed files with 112 additions and 234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
nova/api/openstack/compute/contrib/cloudpipe.py
View File

@ -15,6 +15,7 @@
"""Connect your vlan to the world."""
from oslo_config import cfg
from oslo_utils import fileutils
from oslo_utils import timeutils
from webob import exc
@ -26,7 +27,6 @@ from nova.compute import vm_states
from nova import exception
from nova.i18n import _
from nova import network
from nova.openstack.common import fileutils
from nova import utils
CONF = cfg.CONF

2
nova/api/openstack/compute/plugins/v3/cloudpipe.py
View File

@ -15,6 +15,7 @@
"""Connect your vlan to the world."""
from oslo_config import cfg
from oslo_utils import fileutils
from oslo_utils import timeutils
from webob import exc
@ -30,7 +31,6 @@ from nova import exception
from nova.i18n import _
from nova import network
from nova import objects
from nova.openstack.common import fileutils
from nova import utils
CONF = cfg.CONF

5
nova/cells/state.py
View File

@ -36,7 +36,6 @@ from nova.db import base
from nova import exception
from nova.i18n import _LE
from nova import objects
from nova.openstack.common import fileutils
from nova import rpc
from nova import utils
@ -484,8 +483,8 @@ class CellStateManagerFile(CellStateManager):
:param force: If True, cell status will be updated regardless
of whether it's time to do so.
"""
reloaded, data = fileutils.read_cached_file(self.cells_config_path,
force_reload=force)
reloaded, data = utils.read_cached_file(self.cells_config_path,
force_reload=force)
if reloaded:
LOG.debug("Updating cell cache from config file.")

2
nova/cloudpipe/pipelib.py
View File

@ -26,6 +26,7 @@ import zipfile
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import fileutils
from nova import compute
from nova.compute import flavors
@ -33,7 +34,6 @@ from nova import crypto
from nova import db
from nova import exception
from nova.i18n import _
from nova.openstack.common import fileutils
from nova import paths
from nova import utils

2
nova/crypto.py
View File

@ -32,6 +32,7 @@ from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import excutils
from oslo_utils import fileutils
from oslo_utils import timeutils
import paramiko
from pyasn1.codec.der import encoder as der_encoder
@ -42,7 +43,6 @@ from nova import context
from nova import db
from nova import exception
from nova.i18n import _, _LE
from nova.openstack.common import fileutils
from nova import paths
from nova import utils

2
nova/network/linux_net.py
View File

@ -29,6 +29,7 @@ from oslo_config import cfg
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import excutils
from oslo_utils import fileutils
from oslo_utils import importutils
from oslo_utils import timeutils
import six
@ -36,7 +37,6 @@ import six
from nova import exception
from nova.i18n import _, _LE, _LW
from nova import objects
from nova.openstack.common import fileutils
from nova import paths
from nova.pci import utils as pci_utils
from nova import utils

2
nova/objectstore/s3server.py
View File

@ -41,12 +41,12 @@ import urllib
from oslo_config import cfg
from oslo_log import log as logging
from oslo_log import versionutils
from oslo_utils import fileutils
import routes
import six
import webob
from nova.i18n import _LW
from nova.openstack.common import fileutils
from nova import paths
from nova import utils
from nova import wsgi

149
nova/openstack/common/fileutils.py
View File

@ -1,149 +0,0 @@
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import contextlib
import errno
import logging
import os
import stat
import tempfile
from oslo_utils import excutils
LOG = logging.getLogger(__name__)
_FILE_CACHE = {}
DEFAULT_MODE = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
def ensure_tree(path, mode=DEFAULT_MODE):
"""Create a directory (and any ancestor directories required)
:param path: Directory to create
:param mode: Directory creation permissions
"""
try:
os.makedirs(path, mode)
except OSError as exc:
if exc.errno == errno.EEXIST:
if not os.path.isdir(path):
raise
else:
raise
def read_cached_file(filename, force_reload=False):
"""Read from a file if it has been modified.
:param force_reload: Whether to reload the file.
:returns: A tuple with a boolean specifying if the data is fresh
or not.
"""
global _FILE_CACHE
if force_reload:
delete_cached_file(filename)
reloaded = False
mtime = os.path.getmtime(filename)
cache_info = _FILE_CACHE.setdefault(filename, {})
if not cache_info or mtime > cache_info.get('mtime', 0):
LOG.debug("Reloading cached file %s", filename)
with open(filename) as fap:
cache_info['data'] = fap.read()
cache_info['mtime'] = mtime
reloaded = True
return (reloaded, cache_info['data'])
def delete_cached_file(filename):
"""Delete cached file if present.
:param filename: filename to delete
"""
global _FILE_CACHE
if filename in _FILE_CACHE:
del _FILE_CACHE[filename]
def delete_if_exists(path, remove=os.unlink):
"""Delete a file, but ignore file not found error.
:param path: File to delete
:param remove: Optional function to remove passed path
"""
try:
remove(path)
except OSError as e:
if e.errno != errno.ENOENT:
raise
@contextlib.contextmanager
def remove_path_on_error(path, remove=delete_if_exists):
"""Protect code that wants to operate on PATH atomically.
Any exception will cause PATH to be removed.
:param path: File to work with
:param remove: Optional function to remove passed path
"""
try:
yield
except Exception:
with excutils.save_and_reraise_exception():
remove(path)
def file_open(*args, **kwargs):
"""Open file
see built-in open() documentation for more details
Note: The reason this is kept in a separate module is to easily
be able to provide a stub module that doesn't alter system
state at all (for unit tests)
"""
return open(*args, **kwargs)
def write_to_tempfile(content, path=None, suffix='', prefix='tmp'):
"""Create temporary file or use existing file.
This util is needed for creating temporary file with
specified content, suffix and prefix. If path is not None,
it will be used for writing content. If the path doesn't
exist it'll be created.
:param content: content for temporary file.
:param path: same as parameter 'dir' for mkstemp
:param suffix: same as parameter 'suffix' for mkstemp
:param prefix: same as parameter 'prefix' for mkstemp
For example: it can be used in database tests for creating
configuration files.
"""
if path:
ensure_tree(path)
(fd, path) = tempfile.mkstemp(suffix=suffix, dir=path, prefix=prefix)
try:
os.write(fd, content)
finally:
os.close(fd)
return path

7
nova/openstack/common/policy.py
View File

@ -101,9 +101,8 @@ import six
import six.moves.urllib.parse as urlparse
import six.moves.urllib.request as urlrequest
from nova.openstack.common import fileutils
from nova.openstack.common._i18n import _, _LE
from nova import utils
policy_opts = [
cfg.StrOpt('policy_file',
@ -248,7 +247,7 @@ class Enforcer(object):
def clear(self):
"""Clears Enforcer rules, policy's cache and policy's path."""
self.set_rules({})
fileutils.delete_cached_file(self.policy_path)
utils.delete_cached_file(self.policy_path)
self.default_rule = None
self.policy_path = None
@ -287,7 +286,7 @@ class Enforcer(object):
func(os.path.join(path, policy_file), *args)
def _load_policy_file(self, path, force_reload, overwrite=True):
reloaded, data = fileutils.read_cached_file(
reloaded, data = utils.read_cached_file(
path, force_reload=force_reload)
if reloaded or not self.rules or not overwrite:
rules = Rules.load_json(data, self.default_rule)

4
nova/tests/unit/cells/test_cells_state_manager.py
View File

@ -28,8 +28,8 @@ from nova import db
from nova.db.sqlalchemy import models
from nova import exception
from nova import objects
from nova.openstack.common import fileutils
from nova import test
from nova import utils
FAKE_COMPUTES = [
('host1', 1024, 100, 0, 0),
@ -114,7 +114,7 @@ class TestCellsStateManager(test.NoDBTestCase):
self.assertEqual(['no_such_file_exists.conf'], e.config_files)
@mock.patch.object(cfg.ConfigOpts, 'find_file')
@mock.patch.object(fileutils, 'read_cached_file')
@mock.patch.object(utils, 'read_cached_file')
def test_filemanager_returned(self, mock_read_cached_file, mock_find_file):
mock_find_file.return_value = "/etc/nova/cells.json"
mock_read_cached_file.return_value = (False, six.StringIO({}))

2
nova/tests/unit/network/test_linux_net.py
View File

@ -25,6 +25,7 @@ from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import fileutils
from oslo_utils import timeutils
from nova import context
@ -33,7 +34,6 @@ from nova import exception
from nova.network import driver
from nova.network import linux_net
from nova import objects
from nova.openstack.common import fileutils
from nova import test
from nova import utils

2
nova/tests/unit/test_configdrive2.py
View File

@ -20,9 +20,9 @@ import tempfile
import mock
from mox3 import mox
from oslo_config import cfg
from oslo_utils import fileutils
from nova import context
from nova.openstack.common import fileutils
from nova import test
from nova.tests.unit import fake_instance
from nova import utils

73
nova/tests/unit/test_utils.py
View File

@ -24,7 +24,6 @@ import tempfile
import eventlet
import mock
from mox3 import mox
import netaddr
from oslo_concurrency import processutils
from oslo_config import cfg
@ -95,35 +94,6 @@ class GenericUtilsTestCase(test.NoDBTestCase):
hostname = "<}\x1fh\x10e\x08l\x02l\x05o\x12!{>"
self.assertEqual("hello", utils.sanitize_hostname(hostname))
def test_read_cached_file(self):
self.mox.StubOutWithMock(os.path, "getmtime")
os.path.getmtime(mox.IgnoreArg()).AndReturn(1)
self.mox.ReplayAll()
cache_data = {"data": 1123, "mtime": 1}
data = utils.read_cached_file("/this/is/a/fake", cache_data)
self.assertEqual(cache_data["data"], data)
def test_read_modified_cached_file(self):
self.mox.StubOutWithMock(os.path, "getmtime")
os.path.getmtime(mox.IgnoreArg()).AndReturn(2)
self.mox.ReplayAll()
fake_contents = "lorem ipsum"
m = mock.mock_open(read_data=fake_contents)
with mock.patch("six.moves.builtins.open", m, create=True):
cache_data = {"data": 1123, "mtime": 1}
self.reload_called = False
def test_reload(reloaded_data):
self.assertEqual(reloaded_data, fake_contents)
self.reload_called = True
data = utils.read_cached_file("/this/is/a/fake", cache_data,
reload_func=test_reload)
self.assertEqual(data, fake_contents)
self.assertTrue(self.reload_called)
def test_generate_password(self):
password = utils.generate_password()
self.assertTrue([c for c in password if c in '0123456789'])
@ -235,6 +205,49 @@ class GenericUtilsTestCase(test.NoDBTestCase):
mock_method.assert_called_once_with(*expected_args)
class TestCachedFile(test.NoDBTestCase):
@mock.patch('os.path.getmtime', return_value=1)
def test_read_cached_file(self, getmtime):
utils._FILE_CACHE = {
'/this/is/a/fake': {"data": 1123, "mtime": 1}
}
fresh, data = utils.read_cached_file("/this/is/a/fake")
fdata = utils._FILE_CACHE['/this/is/a/fake']["data"]
self.assertEqual(fdata, data)
@mock.patch('os.path.getmtime', return_value=2)
def test_read_modified_cached_file(self, getmtime):
utils._FILE_CACHE = {
'/this/is/a/fake': {"data": 1123, "mtime": 1}
}
fake_contents = "lorem ipsum"
with mock.patch('six.moves.builtins.open',
mock.mock_open(read_data=fake_contents)):
fresh, data = utils.read_cached_file("/this/is/a/fake")
self.assertEqual(data, fake_contents)
self.assertTrue(fresh)
def test_delete_cached_file(self):
filename = '/this/is/a/fake/deletion/of/cached/file'
utils._FILE_CACHE = {
filename: {"data": 1123, "mtime": 1}
}
self.assertIn(filename, utils._FILE_CACHE)
utils.delete_cached_file(filename)
self.assertNotIn(filename, utils._FILE_CACHE)
def test_delete_cached_file_not_exist(self):
# We expect that if cached file does not exist no Exception raised.
filename = '/this/is/a/fake/deletion/attempt/of/not/cached/file'
self.assertNotIn(filename, utils._FILE_CACHE)
utils.delete_cached_file(filename)
self.assertNotIn(filename, utils._FILE_CACHE)
class VPNPingTestCase(test.NoDBTestCase):
"""Unit tests for utils.vpn_ping()."""
def setUp(self):

2
nova/tests/unit/virt/hyperv/test_hypervapi.py
View File

@ -24,6 +24,7 @@ import uuid
import mock
from mox3 import mox
from oslo_config import cfg
from oslo_utils import fileutils
from oslo_utils import units
import six
@ -33,7 +34,6 @@ from nova import db
from nova import exception
from nova.image import glance
from nova import objects
from nova.openstack.common import fileutils
from nova import test
from nova.tests.unit import fake_flavor
from nova.tests.unit import fake_instance

12
nova/tests/unit/virt/libvirt/test_driver.py
View File

@ -39,6 +39,7 @@ from oslo_config import cfg
from oslo_serialization import jsonutils
from oslo_service import loopingcall
from oslo_utils import encodeutils
from oslo_utils import fileutils
from oslo_utils import importutils
from oslo_utils import timeutils
from oslo_utils import units
@ -59,7 +60,6 @@ from nova import db
from nova import exception
from nova.network import model as network_model
from nova import objects
from nova.openstack.common import fileutils
from nova.pci import manager as pci_manager
from nova import test
from nova.tests.unit import fake_block_device
@ -3126,7 +3126,7 @@ class LibvirtConnTestCase(test.NoDBTestCase):
@mock.patch('nova.virt.disk.api.teardown_container')
@mock.patch('nova.virt.libvirt.driver.LibvirtDriver.get_info')
@mock.patch('nova.virt.disk.api.setup_container')
@mock.patch('nova.openstack.common.fileutils.ensure_tree')
@mock.patch('oslo_utils.fileutils.ensure_tree')
@mock.patch.object(fake_libvirt_utils, 'get_instance_path')
def test_unmount_fs_if_error_during_lxc_create_domain(self,
mock_get_inst_path, mock_ensure_tree, mock_setup_container,
@ -8145,7 +8145,7 @@ class LibvirtConnTestCase(test.NoDBTestCase):
drvr._hard_reboot(self.context, instance, network_info,
block_device_info)
@mock.patch('nova.openstack.common.fileutils.ensure_tree')
@mock.patch('oslo_utils.fileutils.ensure_tree')
@mock.patch('oslo_service.loopingcall.FixedIntervalLoopingCall')
@mock.patch('nova.pci.manager.get_instance_pci_devs')
@mock.patch('nova.virt.libvirt.LibvirtDriver._prepare_pci_devices_for_use')
@ -9972,7 +9972,7 @@ class LibvirtConnTestCase(test.NoDBTestCase):
@mock.patch('nova.virt.disk.api.clean_lxc_namespace')
@mock.patch('nova.virt.libvirt.driver.LibvirtDriver.get_info')
@mock.patch('nova.virt.disk.api.setup_container')
@mock.patch('nova.openstack.common.fileutils.ensure_tree')
@mock.patch('oslo_utils.fileutils.ensure_tree')
@mock.patch.object(fake_libvirt_utils, 'get_instance_path')
def test_create_domain_lxc(self, mock_get_inst_path, mock_ensure_tree,
mock_setup_container, mock_get_info, mock_clean):
@ -10025,7 +10025,7 @@ class LibvirtConnTestCase(test.NoDBTestCase):
@mock.patch('nova.virt.libvirt.driver.LibvirtDriver.get_info')
@mock.patch.object(fake_libvirt_utils, 'chown_for_id_maps')
@mock.patch('nova.virt.disk.api.setup_container')
@mock.patch('nova.openstack.common.fileutils.ensure_tree')
@mock.patch('oslo_utils.fileutils.ensure_tree')
@mock.patch.object(fake_libvirt_utils, 'get_instance_path')
def test_create_domain_lxc_id_maps(self, mock_get_inst_path,
mock_ensure_tree, mock_setup_container,
@ -10092,7 +10092,7 @@ class LibvirtConnTestCase(test.NoDBTestCase):
@mock.patch('nova.virt.disk.api.teardown_container')
@mock.patch('nova.virt.libvirt.driver.LibvirtDriver.get_info')
@mock.patch('nova.virt.disk.api.setup_container')
@mock.patch('nova.openstack.common.fileutils.ensure_tree')
@mock.patch('oslo_utils.fileutils.ensure_tree')
@mock.patch.object(fake_libvirt_utils, 'get_instance_path')
def test_create_domain_lxc_not_running(self, mock_get_inst_path,
mock_ensure_tree,

2
nova/tests/unit/virt/libvirt/test_quobyte.py
View File

@ -18,9 +18,9 @@ import mock
import os
from oslo_concurrency import processutils
from oslo_utils import fileutils
from nova import exception
from nova.openstack.common import fileutils
from nova import test
from nova import utils
from nova.virt.libvirt import quobyte

2
nova/tests/unit/virt/libvirt/test_utils.py
View File

@ -20,11 +20,11 @@ import tempfile
import mock
from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_utils import fileutils
import six
from nova.compute import arch
from nova import exception
from nova.openstack.common import fileutils
from nova.storage import linuxscsi
from nova import test
from nova.tests.unit import fake_instance

59
nova/utils.py
View File

@ -173,6 +173,8 @@ SM_SKIP_KEYS = (
'img_mappings', 'img_block_device_mapping',
)
_FILE_CACHE = {}
def vpn_ping(address, port, timeout=0.05, session_id=None):
"""Sends a vpn negotiation packet and returns the server session.
@ -634,27 +636,6 @@ def sanitize_hostname(hostname):
return hostname
def read_cached_file(filename, cache_info, reload_func=None):
"""Read from a file if it has been modified.
:param cache_info: dictionary to hold opaque cache.
:param reload_func: optional function to be called with data when
file is reloaded due to a modification.
:returns: data from file
"""
mtime = os.path.getmtime(filename)
if not cache_info or mtime != cache_info.get('mtime'):
LOG.debug("Reloading cached file %s", filename)
with open(filename) as fap:
cache_info['data'] = fap.read()
cache_info['mtime'] = mtime
if reload_func:
reload_func(cache_info['data'])
return cache_info['data']
@contextlib.contextmanager
def temporary_mutation(obj, **kwargs):
"""Temporarily set the attr on a particular object to a given value then
@ -1345,3 +1326,39 @@ def safe_truncate(value, length):
except UnicodeDecodeError:
b_value = b_value[:-1]
return u_value
def read_cached_file(filename, force_reload=False):
"""Read from a file if it has been modified.
:param force_reload: Whether to reload the file.
:returns: A tuple with a boolean specifying if the data is fresh
or not.
"""
global _FILE_CACHE
if force_reload:
delete_cached_file(filename)
reloaded = False
mtime = os.path.getmtime(filename)
cache_info = _FILE_CACHE.setdefault(filename, {})
if not cache_info or mtime > cache_info.get('mtime', 0):
LOG.debug("Reloading cached file %s", filename)
with open(filename) as fap:
cache_info['data'] = fap.read()
cache_info['mtime'] = mtime
reloaded = True
return (reloaded, cache_info['data'])
def delete_cached_file(filename):
"""Delete cached file if present.
:param filename: filename to delete
"""
global _FILE_CACHE
if filename in _FILE_CACHE:
del _FILE_CACHE[filename]

2
nova/virt/configdrive.py
View File

@ -20,12 +20,12 @@ import shutil
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import fileutils
from oslo_utils import strutils
from oslo_utils import units
from nova import exception
from nova.i18n import _LW
from nova.openstack.common import fileutils
from nova import utils
from nova import version

2
nova/virt/hyperv/vmops.py
View File

@ -27,6 +27,7 @@ from oslo_config import cfg
from oslo_log import log as logging
from oslo_service import loopingcall
from oslo_utils import excutils
from oslo_utils import fileutils
from oslo_utils import importutils
from oslo_utils import units
from oslo_utils import uuidutils
@ -34,7 +35,6 @@ from oslo_utils import uuidutils
from nova.api.metadata import base as instance_metadata
from nova import exception
from nova.i18n import _, _LI, _LE, _LW
from nova.openstack.common import fileutils
from nova import utils
from nova.virt import configdrive
from nova.virt import hardware

2
nova/virt/images.py
View File

@ -23,11 +23,11 @@ import os
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import fileutils
from nova import exception
from nova.i18n import _, _LE
from nova import image
from nova.openstack.common import fileutils
from nova.openstack.common import imageutils
from nova import utils

2
nova/virt/libvirt/driver.py
View File

@ -49,6 +49,7 @@ from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_service import loopingcall
from oslo_utils import excutils
from oslo_utils import fileutils
from oslo_utils import importutils
from oslo_utils import strutils
from oslo_utils import timeutils
@ -75,7 +76,6 @@ from nova.i18n import _LW
from nova import image
from nova.network import model as network_model
from nova import objects
from nova.openstack.common import fileutils
from nova.pci import manager as pci_manager
from nova.pci import utils as pci_utils
from nova import utils

2
nova/virt/libvirt/imagebackend.py
View File

@ -24,6 +24,7 @@ from oslo_config import cfg
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import excutils
from oslo_utils import fileutils
from oslo_utils import strutils
from oslo_utils import units
import six
@ -33,7 +34,6 @@ from nova.i18n import _
from nova.i18n import _LE, _LI
from nova import image
from nova import keymgr
from nova.openstack.common import fileutils
from nova import utils
from nova.virt.disk import api as disk
from nova.virt.image import model as imgmodel

2
nova/virt/libvirt/imagecache.py
View File

@ -30,11 +30,11 @@ from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import fileutils
from nova.i18n import _LE
from nova.i18n import _LI
from nova.i18n import _LW
from nova.openstack.common import fileutils
from nova import utils
from nova.virt import imagecache
from nova.virt.libvirt import utils as libvirt_utils

2
nova/virt/libvirt/quobyte.py
View File

@ -17,12 +17,12 @@ import os
from oslo_concurrency import processutils
from oslo_log import log as logging
from oslo_utils import fileutils
from nova import exception as nova_exception
from nova.i18n import _
from nova.i18n import _LE
from nova.i18n import _LI
from nova.openstack.common import fileutils
from nova import utils

1
openstack-common.conf
View File

@ -2,7 +2,6 @@
# The list of modules to copy from oslo-incubator
module=cliutils
module=fileutils
module=imageutils
module=memorycache