From a8ca5cb86b4c502e4de93538f1cd9f76fccc127c Mon Sep 17 00:00:00 2001 From: Saad Zaher Date: Wed, 26 Apr 2017 17:41:47 +0000 Subject: [PATCH] Move install-guide to freezer repo * Added install-guide to freezer repo * Remove install-guide from freezer-api Change-Id: I122f2801e6853a2b3c3bc65408b3cec8fe1b555d --- install-guide/source/actions.rst | 27 ++ install-guide/source/api_documents.rst | 12 + install-guide/source/api_routes.rst | 79 ++++ install-guide/source/client_structure.rst | 29 ++ install-guide/source/common_configure.rst | 10 + install-guide/source/common_prerequisites.rst | 75 ++++ install-guide/source/conf.py | 301 +++++++++++++++ install-guide/source/db-install.rst | 53 +++ install-guide/source/devstack_plugin.rst | 48 +++ install-guide/source/get_started.rst | 32 ++ install-guide/source/index.rst | 21 ++ install-guide/source/install-obs.rst | 34 ++ install-guide/source/install-rdo.rst | 33 ++ install-guide/source/install-ubuntu.rst | 31 ++ install-guide/source/install.rst | 175 +++++++++ install-guide/source/jobs.rst | 357 ++++++++++++++++++ install-guide/source/known_issues.rst | 22 ++ install-guide/source/metadata_structure.rst | 55 +++ install-guide/source/next-steps.rst | 9 + install-guide/source/sessions.rst | 104 +++++ install-guide/source/verify.rst | 24 ++ test-requirements.txt | 1 + tox.ini | 11 +- 23 files changed, 1542 insertions(+), 1 deletion(-) create mode 100644 install-guide/source/actions.rst create mode 100644 install-guide/source/api_documents.rst create mode 100644 install-guide/source/api_routes.rst create mode 100644 install-guide/source/client_structure.rst create mode 100644 install-guide/source/common_configure.rst create mode 100644 install-guide/source/common_prerequisites.rst create mode 100644 install-guide/source/conf.py create mode 100644 install-guide/source/db-install.rst create mode 100644 install-guide/source/devstack_plugin.rst create mode 100644 install-guide/source/get_started.rst create mode 100644 install-guide/source/index.rst create mode 100644 install-guide/source/install-obs.rst create mode 100644 install-guide/source/install-rdo.rst create mode 100644 install-guide/source/install-ubuntu.rst create mode 100644 install-guide/source/install.rst create mode 100644 install-guide/source/jobs.rst create mode 100644 install-guide/source/known_issues.rst create mode 100644 install-guide/source/metadata_structure.rst create mode 100644 install-guide/source/next-steps.rst create mode 100644 install-guide/source/sessions.rst create mode 100644 install-guide/source/verify.rst diff --git a/install-guide/source/actions.rst b/install-guide/source/actions.rst new file mode 100644 index 00000000..e0033de7 --- /dev/null +++ b/install-guide/source/actions.rst @@ -0,0 +1,27 @@ +.. _actions: + +Actions +======= + +Actions are stored only to facilitate the assembling of different actions into jobs in the web UI. +They are not directly used by the scheduler. +They are stored in this structure + +.. code-block:: none + + + { + + "freezer_action": { + "action": string, + "backup_name": string, + .... + }, + "mandatory": bool, + "max_retries": int, + "max_retries_interval": int + + "action_id": string, + "user_id": string + } + diff --git a/install-guide/source/api_documents.rst b/install-guide/source/api_documents.rst new file mode 100644 index 00000000..38a23e71 --- /dev/null +++ b/install-guide/source/api_documents.rst @@ -0,0 +1,12 @@ +.. _api_documents: + +API Documents +============= + +Freezer has different types of documents as follow: + +.. toctree:: + + jobs.rst + actions.rst + sessions.rst diff --git a/install-guide/source/api_routes.rst b/install-guide/source/api_routes.rst new file mode 100644 index 00000000..742c203c --- /dev/null +++ b/install-guide/source/api_routes.rst @@ -0,0 +1,79 @@ +.. _api_routes: + +API routes +========== + +General +------- + +.. code-block:: rest + + GET / List API version + GET /v1 JSON Home document, see http://tools.ietf.org/html/draft-nottingham-json-home-03 + +Backup metadata +--------------- + +.. code-block:: rest + + GET /v1/backups(?limit,offset) Lists backups + POST /v1/backups Creates backup entry + + GET /v1/backups/{backup_id} Get backup details + DELETE /v1/backups/{backup_id} Deletes the specified backup + +Freezer clients management +-------------------------- + +.. code-block:: rest + + GET /v1/clients(?limit,offset) Lists registered clients + POST /v1/clients Creates client entry + + GET /v1/clients/{freezerc_id} Get client details + UPDATE /v1/clients/{freezerc_id} Updates the specified client information + DELETE /v1/clients/{freezerc_id} Deletes the specified client information + +Freezer jobs management +----------------------- + +.. code-block:: rest + + GET /v1/jobs(?limit,offset) Lists registered jobs + POST /v1/jobs Creates job entry + + GET /v1/jobs/{jobs_id} Get job details + POST /v1/jobs/{jobs_id} creates or replaces a job entry using the specified job_id + DELETE /v1/jobs/{jobs_id} Deletes the specified job information + PATCH /v1/jobs/{jobs_id} Updates part of the document + +Freezer actions management +-------------------------- + +.. code-block:: rest + + GET /v1/actions(?limit,offset) Lists registered action + POST /v1/actions Creates action entry + + GET /v1/actions/{actions_id} Get action details + POST /v1/actions/{actions_id} creates or replaces a action entry using the specified action_id + DELETE /v1/actions/{actions_id} Deletes the specified action information + PATCH /v1/actions/{actions_id} Updates part of the action document + +Freezer sessions management +--------------------------- + +.. code-block:: rest + + GET /v1/sessions(?limit,offset) Lists registered session + POST /v1/sessions Creates session entry + + GET /v1/sessions/{sessions_id} Get session details + POST /v1/sessions/{sessions_id} creates or replaces a session entry using the specified session_id + DELETE /v1/sessions/{sessions_id} Deletes the specified session information + PATCH /v1/sessions/{sessions_id} Updates part of the session document + + POST /v1/sessions/{sessions_id}/action requests actions (e.g. start/end) upon a specific session + + PUT /v1/sessions/{sessions_id}/jobs/{job_id} adds the job to the session + DELETE /v1/sessions/{sessions_id}/jobs/{job_id} adds the job to the session diff --git a/install-guide/source/client_structure.rst b/install-guide/source/client_structure.rst new file mode 100644 index 00000000..e8b74991 --- /dev/null +++ b/install-guide/source/client_structure.rst @@ -0,0 +1,29 @@ +.. _client_structure: + +Freezer Client document structure +================================= + +Identifies a freezer client for the purpose of sending action + +client_info document contains information relevant for client identification + +.. code-block:: none + + client_info: + { + "client_id": string actually a concatenation "tenant-id_hostname" + "hostname": string + "description": string + "uuid": + } + + +client_type document embeds the client_info and adds user_id + +.. code-block:: none + + client_type : + { + "client" : client_info document, + "user_id": string, # owner of the information (OS X-User-Id, keystone provided, added by api) + } diff --git a/install-guide/source/common_configure.rst b/install-guide/source/common_configure.rst new file mode 100644 index 00000000..d32233d6 --- /dev/null +++ b/install-guide/source/common_configure.rst @@ -0,0 +1,10 @@ +2. Edit the ``/etc/freezer-api/freezer-api.conf`` file and complete the following + actions: + + * In the ``[database]`` section, configure database access: + + .. code-block:: ini + + [database] + ... + connection = mysql+pymysql://freezer-api:FREEZER-API_DBPASS@controller/freezer-api diff --git a/install-guide/source/common_prerequisites.rst b/install-guide/source/common_prerequisites.rst new file mode 100644 index 00000000..7dd2b4d7 --- /dev/null +++ b/install-guide/source/common_prerequisites.rst @@ -0,0 +1,75 @@ +Prerequisites +------------- + +Before you install and configure the Backup service, +you must create a database, service credentials, and API endpoints. + +#. To create the database, complete these steps: + + * Use the database access client to connect to the database + server as the ``root`` user: + + .. code-block:: console + + $ mysql -u root -p + + * Create the ``freezer`` database: + + .. code-block:: console + + CREATE DATABASE freezer; + + * Grant proper access to the ``freezer`` database: + + .. code-block:: console + + GRANT ALL PRIVILEGES ON freezer-api.* TO 'freezer'@'localhost' \ + IDENTIFIED BY 'FREEZER_DBPASS'; + GRANT ALL PRIVILEGES ON freezer.* TO 'freezer'@'%' \ + IDENTIFIED BY 'FREEZER_DBPASS'; + + Replace ``FREEZER_DBPASS`` with a suitable password. + + * Exit the database access client. + + .. code-block:: console + + exit; + +#. Source the ``admin`` credentials to gain access to + admin-only CLI commands: + + .. code-block:: console + + $ . admin-openrc + +#. To create the service credentials, complete these steps: + + * Create the ``freezer`` user: + + .. code-block:: console + + $ openstack user create --domain default --password-prompt freezer + + * Add the ``admin`` role to the ``freezer`` user: + + .. code-block:: console + + $ openstack role add --project service --user freezer admin + + * Create the freezer service entities: + + .. code-block:: console + + $ openstack service create --name freezer --description "Backup" backup + +#. Create the Backup service API endpoints: + + .. code-block:: console + + $ openstack endpoint create --region RegionOne \ + backup public http://controller:9090/vY/%\(tenant_id\)s + $ openstack endpoint create --region RegionOne \ + backup internal http://controller:9090/vY/%\(tenant_id\)s + $ openstack endpoint create --region RegionOne \ + backup admin http://controller:9090/vY/%\(tenant_id\)s diff --git a/install-guide/source/conf.py b/install-guide/source/conf.py new file mode 100644 index 00000000..4d37a554 --- /dev/null +++ b/install-guide/source/conf.py @@ -0,0 +1,301 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import os +# import sys + + +import openstackdocstheme + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +# TODO(ajaeger): enable PDF building, for example add 'rst2pdf.pdfbuilder' +# extensions = + +# Add any paths that contain templates here, relative to this directory. +# templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Installation Guide for Backup Service' +bug_tag = u'install-guide' +copyright = u'2016, OpenStack contributors' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.1' +# The full version, including alpha/beta/rc tags. +release = '0.1' + +# A few variables have to be set for the log-a-bug feature. +# giturl: The location of conf.py on Git. Must be set manually. +# gitsha: The SHA checksum of the bug description. Automatically extracted +# from git log. +# bug_tag: Tag for categorizing the bug. Must be set manually. +# These variables are passed to the logabug code via html_context. +giturl = u'http://git.openstack.org/cgit/openstack/freezer-api/tree/install-guide/source' # noqa +git_cmd = "/usr/bin/git log | head -n1 | cut -f2 -d' '" +gitsha = os.popen(git_cmd).read().strip('\n') +html_context = {"gitsha": gitsha, "bug_tag": bug_tag, + "giturl": giturl, + "bug_project": "freezer-api"} + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ["common_prerequisites.rst", "common_configure.rst"] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'openstackdocs' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +html_theme_path = [openstackdocstheme.get_html_theme_path()] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = [] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# So that we can enable "log-a-bug" links from each output HTML page, this +# variable must be set to a format that includes year, month, day, hours and +# minutes. +html_last_updated_fmt = '%Y-%m-%d %H:%M' + + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +html_use_index = False + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +html_show_sourcelink = False + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'install-guide' + +# If true, publish source files +html_copy_source = False + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # 'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', 'InstallGuide.tex', u'Install Guide', + u'OpenStack contributors', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'installguide', u'Install Guide', + [u'OpenStack contributors'], 1) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'InstallGuide', u'Install Guide', + u'OpenStack contributors', 'InstallGuide', + 'This guide shows OpenStack end users how to install ' + 'an OpenStack cloud.', 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# texinfo_no_detailmenu = False + +# -- Options for Internationalization output ------------------------------ +locale_dirs = ['locale/'] + +# -- Options for PDF output -------------------------------------------------- + +pdf_documents = [ + ('index', u'InstallGuide', u'Install Guide', + u'OpenStack contributors') +] diff --git a/install-guide/source/db-install.rst b/install-guide/source/db-install.rst new file mode 100644 index 00000000..0f225dbf --- /dev/null +++ b/install-guide/source/db-install.rst @@ -0,0 +1,53 @@ +.. _db-install: + +Install and configure database +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Before you install and configure the Backup/Restore service, +you must install the database. + +#. To install elasticsearch on Ubuntu, complete these steps: + + * Install java prerequisites: + + .. code-block:: console + + $ sudo apt-get install -y default-jre-headless + + * Download ``elasticsearch`` version 2.3.0 + + .. code-block:: console + + $ wget https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.3.0/elasticsearch-2.3.0.deb + + * Install ``elasticsearch`` + + .. code-block:: console + + $ sudo dpkg -i elasticsearch-2.3.0.deb + $ sudo update-rc.d elasticsearch defaults 95 10 + + + +#. To install elasticsearch on Fedora, complete these steps: + + * Install java prerequisites: + + .. code-block:: console + + $ sudo yum install -y java-1.8.0-openjdk-headless + + * Download ``elasticsearch`` version 2.3.0 + + .. code-block:: console + + $ wget https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/rpm/elasticsearch/2.3.0/elasticsearch-2.3.0.rpm + + * Install ``elasticsearch`` + + .. code-block:: console + + $ sudo yum install -y elasticsearch-2.3.0.rpm + + + diff --git a/install-guide/source/devstack_plugin.rst b/install-guide/source/devstack_plugin.rst new file mode 100644 index 00000000..2b47aa0b --- /dev/null +++ b/install-guide/source/devstack_plugin.rst @@ -0,0 +1,48 @@ +.. _devstack_plugin: + +Devstack Plugin +=============== + +Edit local.conf +--------------- + +To configure the Freezer API with DevStack, you will need to enable the +freezer-api plugin by adding one line to the [[local|localrc]] section +of your local.conf file: + +.. code-block:: ini + + enable_plugin freezer-api [GITREF] + +where + +.. code-block:: none + + is the URL of a freezer-api repository + [GITREF] is an optional git ref (branch/ref/tag). The default is master. + +For example + +.. code-block:: ini + + enable_plugin freezer-api https://git.openstack.org/openstack/freezer-api.git master + +Plugin Options +-------------- + +The plugin makes use of apache2 by default. +To use the *uwsgi* server set the following environment variable + +.. code-block:: bash + + export FREEZER_API_SERVER_TYPE=uwsgi + +The default port is *9090*. To configure the api to listen on a different port +set the variable `FREEZER_API_PORT`. +For example to make use of port 19090 use + +.. code-block:: bash + + export FREEZER_API_PORT=19090 + +For more information, see `openstack_devstack_plugins_install `_ diff --git a/install-guide/source/get_started.rst b/install-guide/source/get_started.rst new file mode 100644 index 00000000..fc230648 --- /dev/null +++ b/install-guide/source/get_started.rst @@ -0,0 +1,32 @@ +====================================== +Backup/Restore and DR service overview +====================================== +The Backup/Restore and DR service provides an easy way to backup and restore + your OpenStack workloads to different storage. + +The Backup and restore service consists of the following components: + - freezer-api + - freezer-agent + - freezer-scheduler + +The Disaster Recovery service consists of the following components: + - freezer-dr + +The service features a RESTful API, which can be used to maintain the status of +your jobs, backups and metadata. + +This chapter assumes a working setup of OpenStack following the base +Installation Guide. + + +Concepts and definitions +======================== + +*hostname* is _probably_ going to be the host fqdn. + +*backup_id* +defined as UUID of a backup + + +``freezer-api`` service + Accepts and responds to end user API calls... diff --git a/install-guide/source/index.rst b/install-guide/source/index.rst new file mode 100644 index 00000000..cbd34e63 --- /dev/null +++ b/install-guide/source/index.rst @@ -0,0 +1,21 @@ +============================= +Backup/Restore and DR service +============================= + +.. toctree:: + :maxdepth: 2 + + get_started.rst + install.rst + verify.rst + next-steps.rst + api_routes.rst + metadata_structure.rst + client_structure.rst + api_documents.rst + known_issues.rst + + +This chapter assumes a working setup of OpenStack following the +`OpenStack Installation Tutorial +`_. \ No newline at end of file diff --git a/install-guide/source/install-obs.rst b/install-guide/source/install-obs.rst new file mode 100644 index 00000000..bd709488 --- /dev/null +++ b/install-guide/source/install-obs.rst @@ -0,0 +1,34 @@ +.. _install-obs: + + +Install and configure for openSUSE and SUSE Linux Enterprise +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This section describes how to install and configure the Backup service +for openSUSE Leap 42.1 and SUSE Linux Enterprise Server 12 SP1. + +.. include:: common_prerequisites.rst + +Install and configure components +-------------------------------- + +#. Install the packages: + + .. code-block:: console + + # zypper --quiet --non-interactive install + +.. include:: common_configure.rst + + +Finalize installation +--------------------- + +Start the Backup services and configure them to start when +the system boots: + +.. code-block:: console + + # systemctl enable openstack-freezer-api.service + + # systemctl start openstack-freezer-api.service diff --git a/install-guide/source/install-rdo.rst b/install-guide/source/install-rdo.rst new file mode 100644 index 00000000..9dd14579 --- /dev/null +++ b/install-guide/source/install-rdo.rst @@ -0,0 +1,33 @@ +.. _install-rdo: + +Install and configure for Red Hat Enterprise Linux and CentOS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +This section describes how to install and configure the Backup service +for Red Hat Enterprise Linux 7 and CentOS 7. + +.. include:: common_prerequisites.rst + +Install and configure components +-------------------------------- + +#. Install the packages: + + .. code-block:: console + + # yum install + +.. include:: common_configure.rst + +Finalize installation +--------------------- + +Start the Backup services and configure them to start when +the system boots: + +.. code-block:: console + + # systemctl enable openstack-freezer-api.service + + # systemctl start openstack-freezer-api.service diff --git a/install-guide/source/install-ubuntu.rst b/install-guide/source/install-ubuntu.rst new file mode 100644 index 00000000..90e9c509 --- /dev/null +++ b/install-guide/source/install-ubuntu.rst @@ -0,0 +1,31 @@ +.. _install-ubuntu: + +Install and configure for Ubuntu +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This section describes how to install and configure the Backup +service for Ubuntu 14.04 (LTS). + +.. include:: common_prerequisites.rst + +Install and configure components +-------------------------------- + +#. Install the packages: + + .. code-block:: console + + # apt-get update + + # apt-get install + +.. include:: common_configure.rst + +Finalize installation +--------------------- + +Restart the Backup services: + +.. code-block:: console + + # service openstack-freezer-api restart diff --git a/install-guide/source/install.rst b/install-guide/source/install.rst new file mode 100644 index 00000000..7b98dc2b --- /dev/null +++ b/install-guide/source/install.rst @@ -0,0 +1,175 @@ +.. _install: + +Install and configure +~~~~~~~~~~~~~~~~~~~~~ + +This section describes how to install and configure the +Backup service, code-named freezer-api, on the controller node. + +This section assumes that you already have a working OpenStack +environment with at least the following components installed: +.. Keystone + +Note that installation and configuration vary by distribution. + +.. toctree:: + :maxdepth: 2 + + db-install.rst + install-obs.rst + install-rdo.rst + install-ubuntu.rst + +.. code-block:: bash + + # git clone https://git.openstack.org/openstack/freezer-api.git + # cd freezer-api + # pip install ./ + +edit config file +---------------- + +.. code-block:: bash + + # sudo cp etc/freezer/freezer-api.conf /etc/freezer/freezer-api.conf + # sudo cp etc/freezer/freezer-paste.ini /etc/freezer/freezer-paste.ini + # sudo vi /etc/freezer/freezer-api.conf + # sudo vi /etc/freezer/freezer-paste.ini + +setup/configure the db +---------------------- + +The currently supported db is Elasticsearch. In case you are using a dedicated instance +of the server, you'll need to start it. Depending on the OS flavor it might be a: + +.. code-block:: bash + + # service elasticsearch start + +or, on systemd + +.. code-block:: bash + + # systemctl start elasticsearch + +Elasticsearch needs to know what type of data each document's field contains. +This information is contained in the `mapping`, or schema definition. +Elasticsearch will use dynamic mapping to try to guess the field type from +the basic datatypes available in JSON, but some field's properties have to be +explicitly declared to tune the indexing engine. +To do that, use the freezer-manage command: +:: + + # freezer-manage db sync + +You should have updated your configuration files before doing this step. +freezer-manage has the following options: + +* To create the db mappings use the following command:: + + # freezer-manage db sync + +* To update the db mappings using the following command. Update means that you + might have some mappings and you want to update it with a more recent ones + :: + + # freezer-manage db update + +* To remove the db mappings using the following command :: + + # freezer-manage db remove + +* To print the db mappings using the following command :: + + # freezer-manage db show + +* To update your settings (number of replicas) all what you need to do is to + change its value in the configuration file and then run the following command :: + + # freezer-manage db update-settings + +If you provided an invalid number of replicas that will cause problems later on, +so it's highly recommended to make sure that you are using the correct number +of replicas. For more info click here `Elasticsearch_Replicas_instructions `_ + +* To get information about optional additional parameters:: + + # freezer-manage -h + +* If you want to add any additional parameter like --yes or --erase, they should + be before the db option. Check the following examples: + +Wrong Example:: + + # freezer-manage db sync -y -e + +Correct Example:: + + # freezer-manage -y -e db sync + +run simple instance +------------------- + +.. code-block:: bash + + # freezer-api + +examples running using uwsgi +---------------------------- + +.. code-block:: bash + + # uwsgi --http :9090 --need-app --master --module freezer_api.cmd.wsgi:application + + # uwsgi --https :9090,foobar.crt,foobar.key --need-app --master --module freezer_api.cmd.wsgi:application + + +example running freezer-api with apache2 +---------------------------------------- + +.. code-block:: none + + # sudo vi /etc/apache2/sites-enabled/freezer-api.conf + + + WSGIDaemonProcess freezer-api processes=2 threads=2 user=freezer display-name=%{GROUP} + WSGIProcessGroup freezer-api + WSGIApplicationGroup freezer-api + WSGIScriptAlias / /opt/stack/freezer_api/cmd/wsgi.py + + = 2.4> + ErrorLogFormat "%M" + + ErrorLog /var/log/%APACHE_NAME%/freezer-api.log + LogLevel warn + CustomLog /var/log/freezer-api/freezer-api_access.log combined + + + Options Indexes FollowSymLinks MultiViews + Require all granted + AllowOverride None + Order allow,deny + allow from all + LimitRequestBody 102400 + + + + +API registration +================ + +.. code-block:: bash + + # openstack user create --domain default --password-prompt freezer + # openstack role add --project service --user freezer admin + + # openstack service create --name freezer --description "Freezer Backup Service" backup + + # openstack endpoint create --region RegionOne backup public http://freezer_api_publicurl:port + # openstack endpoint create --region RegionOne backup internal http://freezer_api_internalurl:port + # openstack endpoint create --region RegionOne backup admin http://freezer_api_adminurl:port + + +.. toctree:: + + devstack_plugin.rst \ No newline at end of file diff --git a/install-guide/source/jobs.rst b/install-guide/source/jobs.rst new file mode 100644 index 00000000..c5ebb8de --- /dev/null +++ b/install-guide/source/jobs.rst @@ -0,0 +1,357 @@ +.. _jobs: + +Jobs +==== + +A job describes a single action to be executed by a freezer client, for example a backup, or a restore. +It contains the necessary information as if they were provided on the command line. + +A job is stored in the api together with some metadata information such as: +job_id, user_id, client_id, status, scheduling information etc + +Scheduling information enables future/recurrent execution of jobs + +.. code-block:: none + + +---------------------+ + | Job | + +---------------------+ job_actions +--------------+ + | +---------------->| job_action | + | +job_id | 0..* +--------------+ freezer_action + | +client_id | | +mandatory |-------------+ + | +user_id | | +retries | | +----------------+ + | +description | job_schedule +--------------+ +->| freezer_action | + | +---------------+ +----------------+ + | | | +-------------------+ + +---------------------+ +-->| job schedule dict | + +-------------------+ + + +job document structure + +.. code-block:: none + + "job": { + "job_action": { parameters for freezer to execute a specific action } + "job_schedule": { scheduling information } + "job_id": string + "client_id": string + "user_id": string + "description": string + } + + "job_actions": + [ + { + "freezer_action" : + { + "action" : string + "mode" : string + "src_file" : string + "backup_name" : string + "container" : string + ... + }, + "mandatory": False, + "max_retries": 3, + "max_retry_interval": 60 + }, + { + "freezer_action" : + { + ... + }, + "mandatory": False, + "max_retries": 3, + "max_retry_interval": 60 + + } + ] + + "job_schedule": { + "time_created": int (timestamp) + "time_started": int (timestamp) + "time_ended": int (timestamp) + "status": string ["stop", "scheduled", "running", "aborting", "removed"] + "event": string ["", "stop", "start", "abort", "remove"] + "result": string ["", "success", "fail", "aborted"] + + SCHEDULING TIME INFORMATION + } + + +Scheduling Time Information +--------------------------- + +Three types of scheduling can be identified + + * date - used for single run jobs + * interval - periodic jobs, providing an interval value + * cron-like jobs + +Each type has specific parameters which can be given. + +date scheduling +--------------- + +.. code-block:: none + + "schedule_date": : datetime isoformat + +interval scheduling +------------------- + +.. code-block:: none + + "schedule_interval" : "continuous", "N weeks" / "N days" / "N hours" / "N minutes" / "N seconds" + + "schedule_start_date" : datetime isoformat + "schedule_end_date" : datetime isoformat + +cron-like scheduling +-------------------- + +.. code-block:: none + + "schedule_year" : 4 digit year + "schedule_month" : 1-12 + "schedule_day" : 1-31 + "schedule_week" : 1-53 + "schedule_day_of_week": 0-6 or string mon,tue,wed,thu,fri,sat,sun + "schedule_hour" : 0-23 + "schedule_minute" : 0-59 + "schedule_second" : 0-59 + + "schedule_start_date" : datetime isoformat + "schedule_end_date" : datetime isoformat + +Job examples +------------ + +example backup freezer_action + +.. code-block:: none + + "freezer_action": { + "action" : "backup" + "mode" : "fs" + "src_file" : "/home/tylerdurden/project_mayhem" + "backup_name" : "project_mayhem_backup" + "container" : "my_backup_container" + "max_backup_level" : int + "always_backup_level": int + "restart_always_backup": int + "no_incremental" : bool + "encrypt_pass_file" : private_key_file + "log_file" : "/var/log/freezer.log" + "hostname" : false + "max_cpu_priority" : false + } + +example restore freezer_action + +.. code-block:: none + + "freezer_action": { + "action": "restore" + "restore-abs-path": "/home/tylerdurden/project_mayhem" + "container" : "my_backup_container" + "backup_name": "project_mayhem_backup" + "restore-from-host": "another_host" + "max_cpu_priority": true + } + + +example scheduled backup job. +job will be executed once at the provided datetime + +.. code-block:: none + + "job": { + "job_actions": + [ + { + "freezer_action": + { + "action" : "backup", + "mode" : "fs", + "src_file" : "/home/tylerdurden/project_mayhem", + "backup_name" : "project_mayhem_backup", + "container" : "my_backup_container", + } + "exit_status": "fail|success" + "max_retries": int, + "max_retries_interval": secs, + "mandatory": bool + }, + { + action + ... + }, + { + action + ... + } + ], + "job_schedule": + { + "time_created": 1234, + "time_started": 1234, + "time_ended": 0, + "status": "stop | scheduled | running", + "schedule_date": "2015-06-02T16:20:00", + } + "job_id": "blabla", + "client_id": "blabla", + "user_id": "blabla", + "description": "scheduled one shot", + } + + + "job": { + "job_actions": + [ ... ], + "job_schedule": + { + "time_created": 1234, + "time_started": 1234, + "time_ended": 0, + + "status": "stop", + "event": "start" + "schedule_interval" : "1 days" + "schedule_start_date" : "2015-06-02T16:20:00" + }, + "job_id": "4822e482fcbb439189a1ad616ac0a72f", + "client_id": "26b4ea367ac64702868653912e9428cc_freezer.mydomain.myid", + "user_id": "35a322dfb2b14f40bc53a29a14309021", + "description": "daily backup", + } + + +multiple scheduling choices allowed + +.. code-block:: none + + "job": { + "job_actions": + [ ... ], + "job_schedule": + { + "time_created": 1234, + "time_started": 1234, + "time_ended": 0, + "status": "scheduled" + "schedule_month" : "1-6, 9-12" + "schedule_day" : "mon, wed, fri" + "schedule_hour": "03" + "schedule_minute": "25" + } + "job_id": "blabla", + "client_id": "blabla", + "user_id": "blabla", + "description": "daily backup", + } + + +Finished job with result + +.. code-block:: none + + "job": { + "job_actions": [ ... ], + "job_schedule": + { + "time_created": 1234, + "time_started": 1234, + "time_ended": 4321, + "status": "stop", + "event": "", + "result": "success", + "schedule_time": "2015-06-02T16:20:00" + }, + "job_id": "blabla", + "client_id": "blabla", + "user_id": "blabla", + "description": "one shot job", + } + + +Actions default values +---------------------- + +It is possible to define properties that span across multiple actions +This allow not to rewrite values that might be the same in multiple actions. +If properties are specifically set in one action, then the specified value is the one used. + +Example + +.. code-block:: none + + "job": { + "action_defaults": { + "log_file": "/tmp/freezer_tmp_log", + "container": "my_backup_container" + }, + "job_actions": [{ + "freezer_action": { + "action": "backup", + "mode": "fs", + "src_file": "/home/user1/file", + "backup_name": "user1_backup" + } + }, { + "freezer_action": { + "action": "backup", + "mode": "fs", + "src_file": "/home/user2/file", + "backup_name": "user2_backup" + } + }, { + "freezer_action": { + "action": "backup", + "mode": "fs", + "src_file": "/home/user3/file", + "backup_name": "user2_backup", + "log_file": "/home/user3/specific_log_file" + } + }], + "description": "scheduled one shot" + } + + +Is Equivalent to + +.. code-block:: none + + "job": { + "job_actions": [{ + "freezer_action": { + "action": "backup", + "mode": "fs", + "src_file": "/home/user1/file", + "backup_name": "user1_backup", + "log_file": "/tmp/freezer_tmp_log", + "container": "my_backup_container" + } + }, { + "freezer_action": { + "action": "backup", + "mode": "fs", + "src_file": "/home/user2/file", + "backup_name": "user2_backup", + "log_file": "/tmp/freezer_tmp_log", + "container": "my_backup_container" + } + }, { + "freezer_action": { + "action": "backup", + "mode": "fs", + "src_file": "/home/user3/file", + "backup_name": "user2_backup", + "log_file": "/home/user3/specific_log_file", + "container": "my_backup_container" + } + }], + "description": "scheduled one shot" + } diff --git a/install-guide/source/known_issues.rst b/install-guide/source/known_issues.rst new file mode 100644 index 00000000..03cee098 --- /dev/null +++ b/install-guide/source/known_issues.rst @@ -0,0 +1,22 @@ +.. _known_issues: + +Known Issues +============ + +Versions of falcon < 0.1.8 +-------------------------- + +Versions of `falcon `_ prior to 0.1.8 (to be precise, +before `this commit `_) +do not have support for error handlers, which are used internally by freezer-api +to specify the outcomes of various actions. + +The absence of this error handling support means that freezer-api **will not start** +on systems running the following, otherwise supported stable versions of +falcon: + +* 0.1.6 +* 0.1.7 + +falcon 0.1.8, which was released on Jan 14, 2014, and all newer versions support +this functionality. \ No newline at end of file diff --git a/install-guide/source/metadata_structure.rst b/install-guide/source/metadata_structure.rst new file mode 100644 index 00000000..aab6dce0 --- /dev/null +++ b/install-guide/source/metadata_structure.rst @@ -0,0 +1,55 @@ +.. _metadata_structure: + +Backup metadata structure +========================= + +.. note:: + sizes are in MB + +.. code-block:: none + + backup_metadata:= + { + "container": string, + "host_name": string, # fqdn, client has to provide consistent information here ! + "backup_name": string, + "time_stamp": int, + "level": int, + "max_level": int, + "mode" : string, (fs mongo mysql) + "fs_real_path": string, + "vol_snap_path": string, + "total_broken_links" : int, + "total_fs_files" : int, + "total_directories" : int, + "backup_size_uncompressed" : int, + "backup_size_compressed" : int, + "compression_alg": string, (gzip bzip xz) + "encrypted": bool, + "client_os": string + "broken_links" : [string, string, string], + "excluded_files" : [string, string, string] + "cli": string, equivalent cli used when executing the backup ? + "version": string + } + + +The api wraps backup_metadata dictionary with some additional information. +It stores and returns the information provided in this form + +.. code-block:: none + + { + "backup_id": string # backup UUID + "user_id": string, # owner of the backup metadata (OS X-User-Id, keystone provided) + "user_name": string # owner of the backup metadata (OS X-User-Name, keystone provided) + + "backup_metadata": { #--- actual backup_metadata provided + "container": string, + "host_name": string, + "backup_name": string, + "timestamp": int, + ... + } + } + diff --git a/install-guide/source/next-steps.rst b/install-guide/source/next-steps.rst new file mode 100644 index 00000000..ad1bdda7 --- /dev/null +++ b/install-guide/source/next-steps.rst @@ -0,0 +1,9 @@ +.. _next-steps: + +Next steps +~~~~~~~~~~ + +Your OpenStack environment now includes the freezer-api service. + +To add additional services, see +https://docs.openstack.org/project-install-guide/ocata/. diff --git a/install-guide/source/sessions.rst b/install-guide/source/sessions.rst new file mode 100644 index 00000000..42c69e59 --- /dev/null +++ b/install-guide/source/sessions.rst @@ -0,0 +1,104 @@ +.. _sessions: + +Sessions +======== + +A session is a group of jobs which share the same scheduling time. A session is identified +by its **session_id** and has a numeric tag (**session_tag**) which is incremented each time that a new session +is started. +The purpose of the *session_tag* is that of identifying a group of jobs which have been executed +together and which therefore represent a snapshot of a distributed system. + +When a job is added to a session, the scheduling time of the session is copied into the +job data structure, so that any job belonging to the same session will start at the same time. + + +Session Data Structure +---------------------- + +.. code-block:: none + + session = + { + "session_id": string, + "session_tag": int, + "description": string, + "hold_off": int (seconds), + "schedule": { scheduling information, same as jobs }, + "jobs": { 'job_id_1': { + "client_id": string, + "status": string, + "result": string + "time_started": int (timestamp), + "time_ended": int (timestamp), + }, + 'job_id_2': { + "client_id": string, + "status": string, + "result": string + "time_started": int (timestamp), + "time_ended": int (timestamp), + } + } + "time_start": int timestamp, + "time_end": int timestamp, + "time_started": int (timestamp), + "time_ended": int (timestamp), + "status": string "completed" "running", + "result": string "success" "fail", + "user_id": string + } + +Session actions +--------------- + +When the freezer scheduler running on a node wants to start a session, +it sends a POST request to the following endpoint: + +.. code-block:: none + + POST /v1/sessions/{sessions_id}/action + +The body of the request bears the action and parameters + +Session START action +-------------------- + +.. code-block:: none + + { + "start": { + "job_id": "JOB_ID_HERE", + "current_tag": 22 + } + } + +Example of a successful response + +.. code-block:: none + + { + 'result': 'success', + 'session_tag': 23 + } + +Session STOP action +------------------- + +.. code-block:: none + + { + "end": { + "job_id": "JOB_ID_HERE", + "current_tag": 23, + "result": "success|fail" + } + } + +Session-Job association +----------------------- + +.. code-block:: rest + + PUT /v1/sessions/{sessions_id}/jobs/{job_id} adds the job to the session + DELETE /v1/sessions/{sessions_id}/jobs/{job_id} adds the job to the session diff --git a/install-guide/source/verify.rst b/install-guide/source/verify.rst new file mode 100644 index 00000000..2af8a30c --- /dev/null +++ b/install-guide/source/verify.rst @@ -0,0 +1,24 @@ +.. _verify: + +Verify operation +~~~~~~~~~~~~~~~~ + +Verify operation of the Backup service. + +.. note:: + + Perform these commands on the controller node. + +#. Source the ``admin`` project credentials to gain access to + admin-only CLI commands: + + .. code-block:: console + + $ . admin-openrc + +#. List service components to verify successful launch and registration + of each process: + + .. code-block:: console + + $ openstack endpoint list diff --git a/test-requirements.txt b/test-requirements.txt index a392ec0e..074f6750 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -14,6 +14,7 @@ testrepository>=0.0.18 # Apache-2.0/BSD testtools>=1.4.0 # MIT reno>=1.8.0 # Apache-2.0 astroid<1.4.0 # LGPLv2.1 # breaks pylint 1.4.4 +openstackdocstheme>=1.5.0 # Apache-2.0 # Tempest Plugin tempest>=14.0.0 # Apache-2.0 diff --git a/tox.ini b/tox.ini index 7f2a11a8..5911b4f5 100644 --- a/tox.ini +++ b/tox.ini @@ -58,8 +58,11 @@ commands = rm -rf .testrepository [testenv:docs] +whitelist_externals = rm + bash commands = python setup.py build_sphinx + sphinx-build -a -E -W -d install-guide/build/doctrees -b html install-guide/source install-guide/build/html [testenv:pep8] commands = flake8 freezer @@ -72,6 +75,12 @@ ignore = H405,H404,H403,H401 show-source = True exclude = .venv,.tox,dist,doc,test,*egg,releasenotes +[testenv:install-guide] +whitelist_externals = rm + bash +commands = + rm -rf install-guide/build/ + sphinx-build -a -E -W -d install-guide/build/doctrees -b html install-guide/source install-guide/build/html + [testenv:releasenotes] commands = sphinx-build -a -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html -