From 037cf3fa8101443ec2187c773f349d5048e9bdba Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Fri, 22 Sep 2017 09:08:54 +0200 Subject: [PATCH] Fix doc build warnings Enable treading warnings as errors so that no new broken stuff gets merged. Remove obsolete way of doing it, it's a nop. Fix bugs found: * Add missing sphinx modules to properly generate content. * Misformatting of comments and files * Do not index the windows modules, they import modules that do not exist on Linux. * Place api docs in reference/api according to http://specs.openstack.org/openstack/docs-specs/specs/pike/os-manuals-migration.html and reference it. Update .gitignore for this. * Include missing files in tree. Change-Id: I57d3124ac9571f189cd0e10e4cf7e805b8e09045 --- doc/.gitignore | 3 +- doc/source/_static/freezer.conf.sample | 68 ++++++++++++------------- doc/source/cli/freezer-agent.rst | 2 +- doc/source/conf.py | 11 ++++ doc/source/index.rst | 10 ++-- doc/source/reference/index.rst | 7 +++ freezer/engine/engine.py | 36 +++++++------ freezer/engine/rsync/rsync.py | 4 +- freezer/engine/rsyncv2/rsyncv2.py | 4 +- freezer/openstack/restore.py | 3 +- freezer/snapshot/lvm.py | 2 +- freezer/storage/base.py | 17 ++++--- freezer/tests/integration/test_agent.py | 3 +- setup.cfg | 8 +-- 14 files changed, 99 insertions(+), 79 deletions(-) create mode 100644 doc/source/reference/index.rst diff --git a/doc/.gitignore b/doc/.gitignore index 4c2e299e..504de97d 100644 --- a/doc/.gitignore +++ b/doc/.gitignore @@ -1,3 +1,2 @@ build/ -source/ref/ -source/api/ +source/reference/api/ \ No newline at end of file diff --git a/doc/source/_static/freezer.conf.sample b/doc/source/_static/freezer.conf.sample index d6ad0392..53decf6b 100644 --- a/doc/source/_static/freezer.conf.sample +++ b/doc/source/_static/freezer.conf.sample @@ -1,5 +1,39 @@ [DEFAULT] +# +# From freezer-scheduler +# + +# Specifies the client_id used when contacting the service. +# If not specified it will be automatically created +# using the tenant-id and the machine hostname. (string value) +#client_id = + +# Prevents the scheduler from using the api service (boolean value) +#no_api = false + +# Used to store/retrieve files on local storage, including those exchanged with +# the api service. Default value is /etc/freezer/scheduler/conf.d (Env: +# FREEZER_SCHEDULER_CONF_D) (string value) +#jobs_dir = /etc/freezer/scheduler/conf.d + +# Specifies the api-polling interval in seconds. Defaults to 60 seconds +# (integer value) +#interval = 60 + +# Prevents the scheduler from running in daemon mode (boolean value) +#no_daemon = false + +# Initialize freezer scheduler with insecure mode (boolean value) +#insecure = false + +# Allow Freezer Scheduler to deny jobs that execute commands for security +# reasons (boolean value) +#disable_exec = false + +# Number of jobs that can be executed at the same time (integer value) +#concurrent_jobs = 1 + # # From oslo.log # @@ -108,37 +142,3 @@ # Enables or disables fatal status of deprecations. (boolean value) #fatal_deprecations = false - -# -# From scheduler -# - -# Specifies the client_id used when contacting the service. -# If not specified it will be automatically created -# using the tenant-id and the machine hostname. (string value) -#client_id = - -# Prevents the scheduler from using the api service (boolean value) -#no_api = false - -# Used to store/retrieve files on local storage, including those exchanged with -# the api service. Default value is /etc/freezer/scheduler/conf.d (Env: -# FREEZER_SCHEDULER_CONF_D) (string value) -#jobs_dir = /etc/freezer/scheduler/conf.d - -# Specifies the api-polling interval in seconds. Defaults to 60 seconds -# (integer value) -#interval = 60 - -# Prevents the scheduler from running in daemon mode (boolean value) -#no_daemon = false - -# Initialize freezer scheduler with insecure mode (boolean value) -#insecure = false - -# Allow Freezer Scheduler to deny jobs that execute commands for security -# reasons (boolean value) -#disable_exec = false - -# Number of jobs that can be executed at the same time (integer value) -#concurrent_jobs = 1 diff --git a/doc/source/cli/freezer-agent.rst b/doc/source/cli/freezer-agent.rst index a583b19d..fee298c2 100644 --- a/doc/source/cli/freezer-agent.rst +++ b/doc/source/cli/freezer-agent.rst @@ -431,7 +431,7 @@ OPTIONS :Type: string :Default: ```` - Exclude files,given as a PATTERN.Ex: --exclude '*.log' will exclude any file with name ending with .log. Default no exclude + Exclude files, given as a PATTERN.Ex: --exclude '\*.log' will exclude any file with name ending with .log. Default no exclude .. oslo.config:option:: dereference_symlink diff --git a/doc/source/conf.py b/doc/source/conf.py index 6cd53df9..07babf5e 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -41,6 +41,8 @@ import sys extensions = ['openstackdocstheme', 'oslo_config.sphinxconfiggen', + 'oslo_config.sphinxext', + 'sphinx.ext.autodoc', ] config_generator_config_file = ( @@ -99,6 +101,15 @@ bug_tag = 'doc' # # List of directories, relative to source directory, that shouldn't be searched # # for source files. # exclude_trees = [] + +# A list of glob-style patterns that should be excluded when looking for +# source files. They are matched against the source file names relative to the +# source directory, using slashes as directory separators on all platforms. +exclude_patterns = [ + # Missing win32serviceutil module on linux + #'api/freezer.scheduler.win_daemon*', +] + # # # The reST default role (used for this markup: `text`) to use # # for all documents. diff --git a/doc/source/index.rst b/doc/source/index.rst index f40704c3..17cce0e6 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,13 +1,8 @@ -.. Freezer documentation master file, created by - sphinx-quickstart on Thu Feb 4 22:27:35 2016. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - Welcome to Freezer's documentation! =================================== Install Guides ------------ +-------------- .. toctree:: :maxdepth: 2 @@ -20,6 +15,7 @@ User Guides :maxdepth: 2 user/index + cli/index Admin Guides ------------ @@ -34,7 +30,7 @@ Dev Guides :maxdepth: 2 contributor/index - + reference/index About Freezer ============= diff --git a/doc/source/reference/index.rst b/doc/source/reference/index.rst new file mode 100644 index 00000000..e1fe0447 --- /dev/null +++ b/doc/source/reference/index.rst @@ -0,0 +1,7 @@ +Reference +========= + +.. toctree:: + :maxdepth: 2 + + api/autoindex diff --git a/freezer/engine/engine.py b/freezer/engine/engine.py index 07cf1f50..8aaaf0be 100644 --- a/freezer/engine/engine.py +++ b/freezer/engine/engine.py @@ -50,24 +50,28 @@ class BackupEngine(object): This class is an abstraction over all implementations. Workflow: + 1) invoke backup - 1.1) try to download metadata for incremental - 1.2) create a dataflow between backup_stream and storage.write_backup - Backup_stream is producer of data, for tar backup - it creates a gnutar subprocess and start to read data from stdout - Storage write_backup is consumer of data, it creates a thread - that store data in storage. - Both streams communicate in non-blocking mode - 1.3) invoke post_backup - now it uploads metadata file + + 1) try to download metadata for incremental + 2) create a dataflow between backup_stream and storage.write_backup + Backup_stream is producer of data, for tar backup + it creates a gnutar subprocess and start to read data from stdout + Storage write_backup is consumer of data, it creates a thread + that store data in storage. + Both streams communicate in non-blocking mode + 3) invoke post_backup - now it uploads metadata file + 2) restore backup - 2.1) define all incremental backups - 2.2) for each incremental backup create a dataflow between - storage.read_backup and restore_stream - Read_backup is data producer, it reads data chunk by chunk from - the specified storage and pushes the chunks into a queue. - Restore stream is a consumer, that is actually does restore (for - tar it is a thread that creates gnutar subprocess and feeds chunks - to stdin of this thread. + + 1) define all incremental backups + 2) for each incremental backup create a dataflow between + storage.read_backup and restore_stream + Read_backup is data producer, it reads data chunk by chunk from + the specified storage and pushes the chunks into a queue. + Restore stream is a consumer, that is actually does restore (for + tar it is a thread that creates gnutar subprocess and feeds chunks + to stdin of this thread. :type storage: freezer.storage.base.Storage """ diff --git a/freezer/engine/rsync/rsync.py b/freezer/engine/rsync/rsync.py index af457f2b..91dfa708 100644 --- a/freezer/engine/rsync/rsync.py +++ b/freezer/engine/rsync/rsync.py @@ -601,7 +601,7 @@ class RsyncEngine(engine.BackupEngine): Return the meta data as a dict structure and a binary string :param fs_path: file abs path - :param new_level + :param new_level: :return: file data structure """ @@ -741,7 +741,7 @@ class RsyncEngine(engine.BackupEngine): """Compute the file or fs tree path signatures. :param fs_path: - :param manifest_path + :param manifest_path: :param write_queue: :return: """ diff --git a/freezer/engine/rsyncv2/rsyncv2.py b/freezer/engine/rsyncv2/rsyncv2.py index a4b12fe8..e08d2399 100644 --- a/freezer/engine/rsyncv2/rsyncv2.py +++ b/freezer/engine/rsyncv2/rsyncv2.py @@ -151,7 +151,7 @@ class Rsyncv2Engine(engine.BackupEngine): """Restore the provided backup into restore_abs_path. Decrypt backup content if encrypted. - Freezer rsync header data structure: + Freezer rsync header data structure:: [ { 'path': '' (path to file), @@ -652,7 +652,7 @@ class Rsyncv2Engine(engine.BackupEngine): Return blocks of changed data. :param fs_path: - :param manifest_path + :param manifest_path: :param write_queue: :return: """ diff --git a/freezer/openstack/restore.py b/freezer/openstack/restore.py index 9502a17b..17b03ce9 100644 --- a/freezer/openstack/restore.py +++ b/freezer/openstack/restore.py @@ -223,9 +223,10 @@ class RestoreOs(object): 2) Download and upload to glance 3) Create volume from glance 4) Delete + :param restore_from_timestamp: :type restore_from_timestamp: int - :param volume_id - id of attached cinder volume + :param volume_id: - id of attached cinder volume """ (info, image) = self._create_image(volume_id, restore_from_timestamp) length = int(info["x-object-meta-length"]) diff --git a/freezer/snapshot/lvm.py b/freezer/snapshot/lvm.py index 336495e2..c100f600 100644 --- a/freezer/snapshot/lvm.py +++ b/freezer/snapshot/lvm.py @@ -195,7 +195,7 @@ def get_lvm_info(path): where the path is mounted on. :param path: the original file system path where backup needs - to be executed + to be executed :returns: a dict containing the keys 'volgroup', 'srcvol' and 'snap_path' """ diff --git a/freezer/storage/base.py b/freezer/storage/base.py index b3a68ae3..e3568a07 100644 --- a/freezer/storage/base.py +++ b/freezer/storage/base.py @@ -194,14 +194,15 @@ class Backup(object): """ Internal freezer representation of backup. Includes: - name (hostname_backup_name) of backup - timestamp of backup (when it was executed) - level of backup (freezer supports incremental backup) - Completed full backup has level 0 and can be restored without any - additional information. - Levels 1, 2, ... means that our backup is incremental and contains - only smart portion of information (that was actually changed - since the last backup) + + * name (hostname_backup_name) of backup + * timestamp of backup (when it was executed) + * level of backup (freezer supports incremental backup): + Completed full backup has level 0 and can be restored without any + additional information. + Levels 1, 2, ... means that our backup is incremental and contains + only smart portion of information (that was actually changed + since the last backup) """ def __init__(self, engine, hostname_backup_name, diff --git a/freezer/tests/integration/test_agent.py b/freezer/tests/integration/test_agent.py index d96d6c57..e9db3b25 100644 --- a/freezer/tests/integration/test_agent.py +++ b/freezer/tests/integration/test_agent.py @@ -163,11 +163,12 @@ class TestBackupFSLocalstorage(common.TestFS): class TestBackupSSH(common.TestFS): """ Tests are executed if the following env vars are defined: + - FREEZER_TEST_SSH_KEY - FREEZER_TEST_SSH_USERNAME - FREEZER_TEST_SSH_HOST - FREEZER_TEST_CONTAINER - (directory on the remote machine used to store backups) + (directory on the remote machine used to store backups) """ @unittest.skipIf(not common.TestFS.use_ssh, diff --git a/setup.cfg b/setup.cfg index 351553e2..b7ad4623 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,6 +48,7 @@ setup-hooks = source-dir = doc/source build-dir = doc/build all_files = 1 +warning-is-error = 1 [files] packages = @@ -69,7 +70,6 @@ tempest.test_plugins = [pbr] # Have pbr generate the module indexes like sphinx autodoc autodoc_index_modules = True - -# Treat sphinx warnings as errors during the docs build; this helps us keep -# the documentation clean. -warnerrors = true +api_doc_dir = reference/api +autodoc_exclude_modules = + freezer.scheduler.win* \ No newline at end of file