openstack/fuel-octane
Code Issues Proposed changes

Add backup and restore repo action

Browse Source 
Change-Id: I0399ab3748283027aa11b63e92385f8243ac572e
(cherry picked from commit 4ffe94822e)
This commit is contained in:
Sergey Abramov 2016-02-15 13:56:42 +03:00 committed by Yuriy Taraday
parent e4a6d33029
commit a9a6c3bfc6
9 changed files with 286 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
octane/commands/backup.py
View File

@ -23,7 +23,7 @@ from octane.handlers import backup_restore
LOG = logging.getLogger(__name__)
def backup_admin_node(path_to_backup):
def backup(path_to_backup, archivators):
if path_to_backup:
_, ext = os.path.splitext(path_to_backup)
if ext in [".gz", ".bz2"]:
@ -34,14 +34,16 @@ def backup_admin_node(path_to_backup):
else:
tar_obj = tarfile.open(fileobj=sys.stdout, mode="w|")
with contextlib.closing(tar_obj) as archive:
for manager in backup_restore.ARCHIVATORS:
for manager in archivators:
manager(archive).backup()
class BackupCommand(command.Command):
class BaseBackupCommand(command.Command):
archivators = None
def get_parser(self, *args, **kwargs):
parser = super(BackupCommand, self).get_parser(*args, **kwargs)
parser = super(BaseBackupCommand, self).get_parser(*args, **kwargs)
parser.add_argument(
"--to",
type=str,
@ -50,4 +52,15 @@ class BackupCommand(command.Command):
return parser
def take_action(self, parsed_args):
backup_admin_node(parsed_args.path)
assert self.archivators
backup(parsed_args.path, self.archivators)
class BackupCommand(BaseBackupCommand):
archivators = backup_restore.ARCHIVATORS
class BackupRepoCommand(BaseBackupCommand):
archivators = backup_restore.REPO_ARCHIVATORS

23
octane/commands/restore.py
View File

@ -22,19 +22,21 @@ from octane.handlers import backup_restore
LOG = logging.getLogger(__name__)
def restore_admin_node(path_to_backup):
def restore_data(path_to_backup, archivators):
with contextlib.closing(tarfile.open(path_to_backup)) as archive:
archivators = [cls(archive) for cls in backup_restore.ARCHIVATORS]
archivators = [cls(archive) for cls in archivators]
for archivator in archivators:
archivator.pre_restore_check()
for archivator in archivators:
archivator.restore()
class RestoreCommand(command.Command):
class BaseRestoreCommand(command.Command):
archivators = None
def get_parser(self, *args, **kwargs):
parser = super(RestoreCommand, self).get_parser(*args, **kwargs)
parser = super(BaseRestoreCommand, self).get_parser(*args, **kwargs)
parser.add_argument(
"--from",
type=str,
@ -45,6 +47,17 @@ class RestoreCommand(command.Command):
return parser
def take_action(self, parsed_args):
assert self.archivators
if not os.path.isfile(parsed_args.path):
raise ValueError("Invalid path to backup file")
restore_admin_node(parsed_args.path)
restore_data(parsed_args.path, self.archivators)
class RestoreCommand(BaseRestoreCommand):
archivators = backup_restore.ARCHIVATORS
class RestoreRepoCommand(BaseRestoreCommand):
archivators = backup_restore.REPO_ARCHIVATORS

6
octane/handlers/backup_restore/__init__.py
View File

@ -14,6 +14,7 @@ from octane.handlers.backup_restore import astute
from octane.handlers.backup_restore import cobbler
from octane.handlers.backup_restore import fuel_keys
from octane.handlers.backup_restore import fuel_uuid
from octane.handlers.backup_restore import mirrors
from octane.handlers.backup_restore import nailgun_plugins
from octane.handlers.backup_restore import postgres
from octane.handlers.backup_restore import puppet
@ -35,3 +36,8 @@ ARCHIVATORS = [
nailgun_plugins.NailgunPluginsArchivator,
puppet.PuppetApplyHost,
]
REPO_ARCHIVATORS = [
mirrors.MirrorsBackup,
mirrors.RepoBackup,
]

83
octane/handlers/backup_restore/mirrors.py Normal file
View File

@ -0,0 +1,83 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
import os
import urlparse
import yaml
from octane.handlers.backup_restore import base
from octane.util import docker
from octane.util import subprocess
class NaigunWWWBackup(base.PathArchivator):
path = "/var/www/nailgun/"
db = "nailgun"
name = None
sql = None
def _get_values_list(self, data):
raise NotImplementedError
def backup(self):
with open("/etc/fuel/astute.yaml", "r") as current:
current_yaml = yaml.load(current)
ipaddr = current_yaml["ADMIN_NETWORK"]["ipaddress"]
results, _ = docker.run_in_container(
"postgres",
[
"sudo",
"-u",
"postgres",
"psql",
self.db,
"--tuples-only",
"-c",
self.sql
],
stdout=subprocess.PIPE)
results = results.strip()
if not results:
return
rows = results.split("\n")
already_backuped = set()
for line in rows:
data = json.loads(line)
for value in self._get_values_list(data):
if ipaddr in value['uri']:
path = urlparse.urlsplit(value['uri']).path
dir_name = path.lstrip("/").split('/', 1)[0]
if dir_name in already_backuped:
continue
already_backuped.add(dir_name)
path = os.path.join(self.path, dir_name)
self.archive.add(path, self.name)
class MirrorsBackup(NaigunWWWBackup):
name = "mirrors"
sql = "select editable from attributes;"
def _get_values_list(self, data):
return data['repo_setup']['repos']['value']
class RepoBackup(NaigunWWWBackup):
name = "repos"
sql = "select generated from attributes;"
def _get_values_list(self, data):
return data['provision']['image_data'].values()

115
octane/tests/test_archivators.py
View File

@ -10,6 +10,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import mock
import os
import pytest
@ -17,6 +18,7 @@ from octane.handlers.backup_restore import astute
from octane.handlers.backup_restore import cobbler
from octane.handlers.backup_restore import fuel_keys
from octane.handlers.backup_restore import fuel_uuid
from octane.handlers.backup_restore import mirrors
from octane.handlers.backup_restore import nailgun_plugins
from octane.handlers.backup_restore import postgres
from octane.handlers.backup_restore import puppet
@ -131,3 +133,116 @@ def test_nailgun_plugins_backup(mocker, path_exists):
test_archive.add.assert_called_once_with(path, name)
else:
assert not test_archive.add.called
@pytest.mark.parametrize(
"cls, name, sql, ipaddr, sql_output, archive_add_list",
[
(
mirrors.MirrorsBackup,
"mirrors",
"select editable from attributes;",
"127.0.0.1",
'{"repo_setup": {"repos": {"value": ['
'{"uri": "http://127.0.0.1:8080/test_fest"},'
'{"uri": "http://127.0.0.1:8080/test_fest"},'
'{"uri": "http://127.0.0.1:8080/test_fest_2"}'
']}}}',
["test_fest", "test_fest_2"]
),
(
mirrors.MirrorsBackup,
"mirrors",
"select editable from attributes;",
"127.0.0.1",
'{"repo_setup": {"repos": {"value": ['
'{"uri": "http://127.0.0.1:8080/test_fest"},'
'{"uri": "http://127.0.0.1:8080/test_fest"},'
'{"uri": "http://127.0.0.1:8080/test_fest_2"}'
']}}}\n'
'{"repo_setup": {"repos": {"value": ['
'{"uri": "http://127.0.0.1:8080/test_fest"},'
'{"uri": "http://127.0.0.1:8080/test_fest_3"},'
'{"uri": "http://127.0.0.1:8080/test_fest_2"}'
']}}}',
["test_fest", "test_fest_2", "test_fest_3"]
),
(
mirrors.MirrorsBackup,
"mirrors",
"select editable from attributes;",
"127.0.0.1",
'',
[]
),
(
mirrors.RepoBackup,
"repos",
"select generated from attributes;",
"127.0.0.1",
'{"provision": {"image_data": {'
'"1": {"uri": "http://127.0.0.1:8080/test_fest"},'
'"2": {"uri": "http://127.0.0.1:8080/test_fest_2"},'
'"3": {"uri": "http://127.0.0.1:8080/test_fest_3"},'
'"4": {"uri": "http://127.0.0.1:8080/test_fest_5"}'
'}}}',
['test_fest', 'test_fest_2', 'test_fest_3', "test_fest_5"]
),
(
mirrors.RepoBackup,
"repos",
"select generated from attributes;",
"127.0.0.1",
'{"provision": {"image_data": {'
'"1": {"uri": "http://127.0.0.1:8080/test_fest"},'
'"2": {"uri": "http://127.0.0.1:8080/test_fest_2"},'
'"3": {"uri": "http://127.0.0.1:8080/test_fest_3"},'
'"4": {"uri": "http://127.0.0.1:8080/test_fest"}'
'}}}\n'
'{"provision": {"image_data": {'
'"1": {"uri": "http://127.0.0.1:8080/test_fest"},'
'"2": {"uri": "http://127.0.0.1:8080/test_fest_2"},'
'"3": {"uri": "http://127.0.0.1:8080/test_fest_3"},'
'"4": {"uri": "http://127.0.0.1:8080/test_fest_5"}'
'}}}',
['test_fest', 'test_fest_2', 'test_fest_3', "test_fest_5"]
),
(
mirrors.RepoBackup,
"repos",
"select generated from attributes;",
"127.0.0.1",
'',
[]
),
]
)
def test_repos_backup(
mocker, mock_open, cls, name, sql, ipaddr,
sql_output, archive_add_list):
yaml_mocker = mocker.patch(
"yaml.load",
return_value={"ADMIN_NETWORK": {"ipaddress": "127.0.0.1"}})
docker_mock = mocker.patch("octane.util.docker.run_in_container")
test_archive = mocker.Mock()
path = "/var/www/nailgun/"
docker_mock.return_value = sql_output, None
cls(test_archive).backup()
yaml_mocker.assert_called_once_with(mock_open.return_value)
docker_mock.assert_called_once_with(
"postgres", [
"sudo",
"-u",
"postgres",
"psql",
"nailgun",
"--tuples-only",
"-c",
sql
],
stdout=subprocess.PIPE
)
test_archive.add.assert_has_calls(
[mock.call(os.path.join(path, i), name) for i in archive_add_list],
any_order=True)
assert test_archive.add.call_count == len(archive_add_list)

19
octane/tests/test_archivators_restore.py
View File

@ -20,6 +20,7 @@ from octane.handlers.backup_restore import astute
from octane.handlers.backup_restore import cobbler
from octane.handlers.backup_restore import fuel_keys
from octane.handlers.backup_restore import fuel_uuid
from octane.handlers.backup_restore import mirrors
from octane.handlers.backup_restore import postgres
from octane.handlers.backup_restore import puppet
from octane.handlers.backup_restore import ssh
@ -114,6 +115,24 @@ class TestArchive(object):
("fuel_uuid/fuel-uuid", True, True),
]
),
(
mirrors.MirrorsBackup,
"/var/www/nailgun/",
[
("mirrors/", False, False),
("mirrors/data.txt", True, True),
("mirrors/subdir/data.txt", True, True),
],
),
(
mirrors.RepoBackup,
"/var/www/nailgun/",
[
("repos/", False, False),
("repos/data.txt", True, True),
("repos/subdir/data.txt", True, True),
],
),
])
def test_path_restore(mocker, cls, path, members):
members = [TestMember(n, f, e) for n, f, e in members]

29
octane/tests/test_backup.py
View File

@ -13,24 +13,24 @@ import pytest
import sys
from octane.commands import backup
from octane.handlers import backup_restore
def test_parser_empty(mocker, octane_app):
m1 = mocker.patch('octane.commands.backup.backup_admin_node')
@pytest.mark.parametrize("cmd,archivators", [
("fuel-backup", backup_restore.ARCHIVATORS),
("fuel-repo-backup", backup_restore.REPO_ARCHIVATORS),
])
@pytest.mark.parametrize("path", [None, "backup_file"])
def test_parser_empty(mocker, octane_app, cmd, archivators, path):
m1 = mocker.patch('octane.commands.backup.backup')
m1.return_value = 2
octane_app.run(["fuel-backup"])
params = [cmd]
if path:
params += ["--to", path]
octane_app.run(params)
assert not octane_app.stdout.getvalue()
assert not octane_app.stderr.getvalue()
m1.assert_called_once_with(None)
def test_parser_not_empty(mocker, octane_app):
m1 = mocker.patch('octane.commands.backup.backup_admin_node')
m1.return_value = 2
octane_app.run(["fuel-backup", "--to", "backup_file"])
assert not octane_app.stdout.getvalue()
assert not octane_app.stderr.getvalue()
m1.assert_called_once_with("backup_file")
m1.assert_called_once_with(path, archivators)
@pytest.mark.parametrize("path,mode", [
@ -42,9 +42,8 @@ def test_parser_not_empty(mocker, octane_app):
])
def test_backup_admin_node_backup_file(mocker, path, mode):
manager = mocker.Mock()
mocker.patch('octane.handlers.backup_restore.ARCHIVATORS', new=[manager])
tar_obj = mocker.patch("tarfile.open")
backup.backup_admin_node(path)
backup.backup(path, [manager])
manager.assert_called_once_with(tar_obj.return_value)
manager.return_value.backup.assert_called_once_with()
if path is not None:

21
octane/tests/test_restore.py
View File

@ -14,6 +14,7 @@ import mock
import pytest
from octane.commands import restore
from octane.handlers import backup_restore
@pytest.mark.parametrize("path,is_file", [
@ -21,10 +22,14 @@ from octane.commands import restore
("path", False),
("path", True),
])
def test_parser(mocker, octane_app, path, is_file):
restore_mock = mocker.patch('octane.commands.restore.restore_admin_node')
@pytest.mark.parametrize("command, archivators", [
("fuel-restore", backup_restore.ARCHIVATORS),
("fuel-repo-restore", backup_restore.REPO_ARCHIVATORS),
])
def test_parser(mocker, octane_app, path, is_file, command, archivators):
restore_mock = mocker.patch('octane.commands.restore.restore_data')
mocker.patch("os.path.isfile", return_value=is_file)
params = ["fuel-restore"]
params = [command]
if path:
params += ["--from", path]
try:
@ -36,21 +41,17 @@ def test_parser(mocker, octane_app, path, is_file):
assert not restore_mock.called
assert not is_file
else:
restore_mock.assert_called_once_with(path)
restore_mock.assert_called_once_with(path, archivators)
assert path is not None
assert is_file
def test_restore_admin_node(mocker):
def test_restore_data(mocker):
tar_mock = mocker.patch("tarfile.open")
archivator_mock_1 = mocker.Mock()
archivator_mock_2 = mocker.Mock()
mocker.patch(
"octane.handlers.backup_restore.ARCHIVATORS",
new=[archivator_mock_1, archivator_mock_2]
)
path = "path"
restore.restore_admin_node(path)
restore.restore_data(path, [archivator_mock_1, archivator_mock_2])
tar_mock.assert_called_once_with(path)
for arch_mock in [archivator_mock_1, archivator_mock_2]:
arch_mock.assert_has_calls([

2
setup.cfg
View File

@ -50,6 +50,8 @@ octane =
update-plugin-settings = octane.commands.update_plugin_settings:UpdatePluginSettingsCommand [zabbix]
fuel-backup = octane.commands.backup:BackupCommand
fuel-restore = octane.commands.restore:RestoreCommand
fuel-repo-backup = octane.commands.backup:BackupRepoCommand
fuel-repo-restore = octane.commands.restore:RestoreRepoCommand
octane.handlers.upgrade =
controller = octane.handlers.upgrade.controller:ControllerUpgrade
compute = octane.handlers.upgrade.compute:ComputeUpgrade