From a9a6c3bfc67dda198ab4fff8d8ea18addb962cf7 Mon Sep 17 00:00:00 2001 From: Sergey Abramov Date: Mon, 15 Feb 2016 13:56:42 +0300 Subject: [PATCH] Add backup and restore repo action Change-Id: I0399ab3748283027aa11b63e92385f8243ac572e (cherry picked from commit 4ffe94822ea41f7d5270e805f16242576264e958) --- octane/commands/backup.py | 23 ++++- octane/commands/restore.py | 23 ++++- octane/handlers/backup_restore/__init__.py | 6 ++ octane/handlers/backup_restore/mirrors.py | 83 +++++++++++++++ octane/tests/test_archivators.py | 115 +++++++++++++++++++++ octane/tests/test_archivators_restore.py | 19 ++++ octane/tests/test_backup.py | 29 +++--- octane/tests/test_restore.py | 21 ++-- setup.cfg | 2 + 9 files changed, 286 insertions(+), 35 deletions(-) create mode 100644 octane/handlers/backup_restore/mirrors.py diff --git a/octane/commands/backup.py b/octane/commands/backup.py index 71209b04..7b947a0f 100644 --- a/octane/commands/backup.py +++ b/octane/commands/backup.py @@ -23,7 +23,7 @@ from octane.handlers import backup_restore LOG = logging.getLogger(__name__) -def backup_admin_node(path_to_backup): +def backup(path_to_backup, archivators): if path_to_backup: _, ext = os.path.splitext(path_to_backup) if ext in [".gz", ".bz2"]: @@ -34,14 +34,16 @@ def backup_admin_node(path_to_backup): else: tar_obj = tarfile.open(fileobj=sys.stdout, mode="w|") with contextlib.closing(tar_obj) as archive: - for manager in backup_restore.ARCHIVATORS: + for manager in archivators: manager(archive).backup() -class BackupCommand(command.Command): +class BaseBackupCommand(command.Command): + + archivators = None def get_parser(self, *args, **kwargs): - parser = super(BackupCommand, self).get_parser(*args, **kwargs) + parser = super(BaseBackupCommand, self).get_parser(*args, **kwargs) parser.add_argument( "--to", type=str, @@ -50,4 +52,15 @@ class BackupCommand(command.Command): return parser def take_action(self, parsed_args): - backup_admin_node(parsed_args.path) + assert self.archivators + backup(parsed_args.path, self.archivators) + + +class BackupCommand(BaseBackupCommand): + + archivators = backup_restore.ARCHIVATORS + + +class BackupRepoCommand(BaseBackupCommand): + + archivators = backup_restore.REPO_ARCHIVATORS diff --git a/octane/commands/restore.py b/octane/commands/restore.py index c79180da..009b28dc 100644 --- a/octane/commands/restore.py +++ b/octane/commands/restore.py @@ -22,19 +22,21 @@ from octane.handlers import backup_restore LOG = logging.getLogger(__name__) -def restore_admin_node(path_to_backup): +def restore_data(path_to_backup, archivators): with contextlib.closing(tarfile.open(path_to_backup)) as archive: - archivators = [cls(archive) for cls in backup_restore.ARCHIVATORS] + archivators = [cls(archive) for cls in archivators] for archivator in archivators: archivator.pre_restore_check() for archivator in archivators: archivator.restore() -class RestoreCommand(command.Command): +class BaseRestoreCommand(command.Command): + + archivators = None def get_parser(self, *args, **kwargs): - parser = super(RestoreCommand, self).get_parser(*args, **kwargs) + parser = super(BaseRestoreCommand, self).get_parser(*args, **kwargs) parser.add_argument( "--from", type=str, @@ -45,6 +47,17 @@ class RestoreCommand(command.Command): return parser def take_action(self, parsed_args): + assert self.archivators if not os.path.isfile(parsed_args.path): raise ValueError("Invalid path to backup file") - restore_admin_node(parsed_args.path) + restore_data(parsed_args.path, self.archivators) + + +class RestoreCommand(BaseRestoreCommand): + + archivators = backup_restore.ARCHIVATORS + + +class RestoreRepoCommand(BaseRestoreCommand): + + archivators = backup_restore.REPO_ARCHIVATORS diff --git a/octane/handlers/backup_restore/__init__.py b/octane/handlers/backup_restore/__init__.py index 5f2340e7..190024d7 100644 --- a/octane/handlers/backup_restore/__init__.py +++ b/octane/handlers/backup_restore/__init__.py @@ -14,6 +14,7 @@ from octane.handlers.backup_restore import astute from octane.handlers.backup_restore import cobbler from octane.handlers.backup_restore import fuel_keys from octane.handlers.backup_restore import fuel_uuid +from octane.handlers.backup_restore import mirrors from octane.handlers.backup_restore import nailgun_plugins from octane.handlers.backup_restore import postgres from octane.handlers.backup_restore import puppet @@ -35,3 +36,8 @@ ARCHIVATORS = [ nailgun_plugins.NailgunPluginsArchivator, puppet.PuppetApplyHost, ] + +REPO_ARCHIVATORS = [ + mirrors.MirrorsBackup, + mirrors.RepoBackup, +] diff --git a/octane/handlers/backup_restore/mirrors.py b/octane/handlers/backup_restore/mirrors.py new file mode 100644 index 00000000..4340473a --- /dev/null +++ b/octane/handlers/backup_restore/mirrors.py @@ -0,0 +1,83 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import os +import urlparse +import yaml + +from octane.handlers.backup_restore import base + +from octane.util import docker +from octane.util import subprocess + + +class NaigunWWWBackup(base.PathArchivator): + path = "/var/www/nailgun/" + db = "nailgun" + name = None + sql = None + + def _get_values_list(self, data): + raise NotImplementedError + + def backup(self): + with open("/etc/fuel/astute.yaml", "r") as current: + current_yaml = yaml.load(current) + ipaddr = current_yaml["ADMIN_NETWORK"]["ipaddress"] + results, _ = docker.run_in_container( + "postgres", + [ + "sudo", + "-u", + "postgres", + "psql", + self.db, + "--tuples-only", + "-c", + self.sql + ], + stdout=subprocess.PIPE) + results = results.strip() + if not results: + return + rows = results.split("\n") + already_backuped = set() + for line in rows: + data = json.loads(line) + for value in self._get_values_list(data): + if ipaddr in value['uri']: + path = urlparse.urlsplit(value['uri']).path + dir_name = path.lstrip("/").split('/', 1)[0] + if dir_name in already_backuped: + continue + already_backuped.add(dir_name) + path = os.path.join(self.path, dir_name) + self.archive.add(path, self.name) + + +class MirrorsBackup(NaigunWWWBackup): + + name = "mirrors" + sql = "select editable from attributes;" + + def _get_values_list(self, data): + return data['repo_setup']['repos']['value'] + + +class RepoBackup(NaigunWWWBackup): + + name = "repos" + sql = "select generated from attributes;" + + def _get_values_list(self, data): + return data['provision']['image_data'].values() diff --git a/octane/tests/test_archivators.py b/octane/tests/test_archivators.py index bbc8cd29..330d7402 100644 --- a/octane/tests/test_archivators.py +++ b/octane/tests/test_archivators.py @@ -10,6 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. +import mock import os import pytest @@ -17,6 +18,7 @@ from octane.handlers.backup_restore import astute from octane.handlers.backup_restore import cobbler from octane.handlers.backup_restore import fuel_keys from octane.handlers.backup_restore import fuel_uuid +from octane.handlers.backup_restore import mirrors from octane.handlers.backup_restore import nailgun_plugins from octane.handlers.backup_restore import postgres from octane.handlers.backup_restore import puppet @@ -131,3 +133,116 @@ def test_nailgun_plugins_backup(mocker, path_exists): test_archive.add.assert_called_once_with(path, name) else: assert not test_archive.add.called + + +@pytest.mark.parametrize( + "cls, name, sql, ipaddr, sql_output, archive_add_list", + [ + ( + mirrors.MirrorsBackup, + "mirrors", + "select editable from attributes;", + "127.0.0.1", + '{"repo_setup": {"repos": {"value": [' + '{"uri": "http://127.0.0.1:8080/test_fest"},' + '{"uri": "http://127.0.0.1:8080/test_fest"},' + '{"uri": "http://127.0.0.1:8080/test_fest_2"}' + ']}}}', + ["test_fest", "test_fest_2"] + ), + ( + mirrors.MirrorsBackup, + "mirrors", + "select editable from attributes;", + "127.0.0.1", + '{"repo_setup": {"repos": {"value": [' + '{"uri": "http://127.0.0.1:8080/test_fest"},' + '{"uri": "http://127.0.0.1:8080/test_fest"},' + '{"uri": "http://127.0.0.1:8080/test_fest_2"}' + ']}}}\n' + '{"repo_setup": {"repos": {"value": [' + '{"uri": "http://127.0.0.1:8080/test_fest"},' + '{"uri": "http://127.0.0.1:8080/test_fest_3"},' + '{"uri": "http://127.0.0.1:8080/test_fest_2"}' + ']}}}', + ["test_fest", "test_fest_2", "test_fest_3"] + ), + ( + mirrors.MirrorsBackup, + "mirrors", + "select editable from attributes;", + "127.0.0.1", + '', + [] + ), + ( + mirrors.RepoBackup, + "repos", + "select generated from attributes;", + "127.0.0.1", + '{"provision": {"image_data": {' + '"1": {"uri": "http://127.0.0.1:8080/test_fest"},' + '"2": {"uri": "http://127.0.0.1:8080/test_fest_2"},' + '"3": {"uri": "http://127.0.0.1:8080/test_fest_3"},' + '"4": {"uri": "http://127.0.0.1:8080/test_fest_5"}' + '}}}', + ['test_fest', 'test_fest_2', 'test_fest_3', "test_fest_5"] + ), + ( + mirrors.RepoBackup, + "repos", + "select generated from attributes;", + "127.0.0.1", + '{"provision": {"image_data": {' + '"1": {"uri": "http://127.0.0.1:8080/test_fest"},' + '"2": {"uri": "http://127.0.0.1:8080/test_fest_2"},' + '"3": {"uri": "http://127.0.0.1:8080/test_fest_3"},' + '"4": {"uri": "http://127.0.0.1:8080/test_fest"}' + '}}}\n' + '{"provision": {"image_data": {' + '"1": {"uri": "http://127.0.0.1:8080/test_fest"},' + '"2": {"uri": "http://127.0.0.1:8080/test_fest_2"},' + '"3": {"uri": "http://127.0.0.1:8080/test_fest_3"},' + '"4": {"uri": "http://127.0.0.1:8080/test_fest_5"}' + '}}}', + ['test_fest', 'test_fest_2', 'test_fest_3', "test_fest_5"] + ), + ( + mirrors.RepoBackup, + "repos", + "select generated from attributes;", + "127.0.0.1", + '', + [] + ), + ] +) +def test_repos_backup( + mocker, mock_open, cls, name, sql, ipaddr, + sql_output, archive_add_list): + yaml_mocker = mocker.patch( + "yaml.load", + return_value={"ADMIN_NETWORK": {"ipaddress": "127.0.0.1"}}) + docker_mock = mocker.patch("octane.util.docker.run_in_container") + test_archive = mocker.Mock() + path = "/var/www/nailgun/" + docker_mock.return_value = sql_output, None + cls(test_archive).backup() + yaml_mocker.assert_called_once_with(mock_open.return_value) + docker_mock.assert_called_once_with( + "postgres", [ + "sudo", + "-u", + "postgres", + "psql", + "nailgun", + "--tuples-only", + "-c", + sql + ], + stdout=subprocess.PIPE + ) + test_archive.add.assert_has_calls( + [mock.call(os.path.join(path, i), name) for i in archive_add_list], + any_order=True) + assert test_archive.add.call_count == len(archive_add_list) diff --git a/octane/tests/test_archivators_restore.py b/octane/tests/test_archivators_restore.py index 50816da7..aac43569 100644 --- a/octane/tests/test_archivators_restore.py +++ b/octane/tests/test_archivators_restore.py @@ -20,6 +20,7 @@ from octane.handlers.backup_restore import astute from octane.handlers.backup_restore import cobbler from octane.handlers.backup_restore import fuel_keys from octane.handlers.backup_restore import fuel_uuid +from octane.handlers.backup_restore import mirrors from octane.handlers.backup_restore import postgres from octane.handlers.backup_restore import puppet from octane.handlers.backup_restore import ssh @@ -114,6 +115,24 @@ class TestArchive(object): ("fuel_uuid/fuel-uuid", True, True), ] ), + ( + mirrors.MirrorsBackup, + "/var/www/nailgun/", + [ + ("mirrors/", False, False), + ("mirrors/data.txt", True, True), + ("mirrors/subdir/data.txt", True, True), + ], + ), + ( + mirrors.RepoBackup, + "/var/www/nailgun/", + [ + ("repos/", False, False), + ("repos/data.txt", True, True), + ("repos/subdir/data.txt", True, True), + ], + ), ]) def test_path_restore(mocker, cls, path, members): members = [TestMember(n, f, e) for n, f, e in members] diff --git a/octane/tests/test_backup.py b/octane/tests/test_backup.py index 8db06590..7407469e 100644 --- a/octane/tests/test_backup.py +++ b/octane/tests/test_backup.py @@ -13,24 +13,24 @@ import pytest import sys from octane.commands import backup +from octane.handlers import backup_restore -def test_parser_empty(mocker, octane_app): - m1 = mocker.patch('octane.commands.backup.backup_admin_node') +@pytest.mark.parametrize("cmd,archivators", [ + ("fuel-backup", backup_restore.ARCHIVATORS), + ("fuel-repo-backup", backup_restore.REPO_ARCHIVATORS), +]) +@pytest.mark.parametrize("path", [None, "backup_file"]) +def test_parser_empty(mocker, octane_app, cmd, archivators, path): + m1 = mocker.patch('octane.commands.backup.backup') m1.return_value = 2 - octane_app.run(["fuel-backup"]) + params = [cmd] + if path: + params += ["--to", path] + octane_app.run(params) assert not octane_app.stdout.getvalue() assert not octane_app.stderr.getvalue() - m1.assert_called_once_with(None) - - -def test_parser_not_empty(mocker, octane_app): - m1 = mocker.patch('octane.commands.backup.backup_admin_node') - m1.return_value = 2 - octane_app.run(["fuel-backup", "--to", "backup_file"]) - assert not octane_app.stdout.getvalue() - assert not octane_app.stderr.getvalue() - m1.assert_called_once_with("backup_file") + m1.assert_called_once_with(path, archivators) @pytest.mark.parametrize("path,mode", [ @@ -42,9 +42,8 @@ def test_parser_not_empty(mocker, octane_app): ]) def test_backup_admin_node_backup_file(mocker, path, mode): manager = mocker.Mock() - mocker.patch('octane.handlers.backup_restore.ARCHIVATORS', new=[manager]) tar_obj = mocker.patch("tarfile.open") - backup.backup_admin_node(path) + backup.backup(path, [manager]) manager.assert_called_once_with(tar_obj.return_value) manager.return_value.backup.assert_called_once_with() if path is not None: diff --git a/octane/tests/test_restore.py b/octane/tests/test_restore.py index d43147f4..b01225af 100644 --- a/octane/tests/test_restore.py +++ b/octane/tests/test_restore.py @@ -14,6 +14,7 @@ import mock import pytest from octane.commands import restore +from octane.handlers import backup_restore @pytest.mark.parametrize("path,is_file", [ @@ -21,10 +22,14 @@ from octane.commands import restore ("path", False), ("path", True), ]) -def test_parser(mocker, octane_app, path, is_file): - restore_mock = mocker.patch('octane.commands.restore.restore_admin_node') +@pytest.mark.parametrize("command, archivators", [ + ("fuel-restore", backup_restore.ARCHIVATORS), + ("fuel-repo-restore", backup_restore.REPO_ARCHIVATORS), +]) +def test_parser(mocker, octane_app, path, is_file, command, archivators): + restore_mock = mocker.patch('octane.commands.restore.restore_data') mocker.patch("os.path.isfile", return_value=is_file) - params = ["fuel-restore"] + params = [command] if path: params += ["--from", path] try: @@ -36,21 +41,17 @@ def test_parser(mocker, octane_app, path, is_file): assert not restore_mock.called assert not is_file else: - restore_mock.assert_called_once_with(path) + restore_mock.assert_called_once_with(path, archivators) assert path is not None assert is_file -def test_restore_admin_node(mocker): +def test_restore_data(mocker): tar_mock = mocker.patch("tarfile.open") archivator_mock_1 = mocker.Mock() archivator_mock_2 = mocker.Mock() - mocker.patch( - "octane.handlers.backup_restore.ARCHIVATORS", - new=[archivator_mock_1, archivator_mock_2] - ) path = "path" - restore.restore_admin_node(path) + restore.restore_data(path, [archivator_mock_1, archivator_mock_2]) tar_mock.assert_called_once_with(path) for arch_mock in [archivator_mock_1, archivator_mock_2]: arch_mock.assert_has_calls([ diff --git a/setup.cfg b/setup.cfg index 1c74c02e..0cfa57c1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,6 +50,8 @@ octane = update-plugin-settings = octane.commands.update_plugin_settings:UpdatePluginSettingsCommand [zabbix] fuel-backup = octane.commands.backup:BackupCommand fuel-restore = octane.commands.restore:RestoreCommand + fuel-repo-backup = octane.commands.backup:BackupRepoCommand + fuel-repo-restore = octane.commands.restore:RestoreRepoCommand octane.handlers.upgrade = controller = octane.handlers.upgrade.controller:ControllerUpgrade compute = octane.handlers.upgrade.compute:ComputeUpgrade