openstack
/
fuel-octane
Code Issues Proposed changes

Remove containers specifics from DB archivators 

Since nailgun and keystone services are not isolated in containers their
processes should be stopped before to manipulate with databases and
data.

DB archivators are collected in DatabasesArchivator to apply the
'postgresql' task before to restore DB states. This task is needed to
configure databases, users and privileges according new parameters in
astute.yaml.

Some new functions were introduced to perform respective operations on
the host level, such as:
  - archivate.archivate_cmd_output
  - sql.run_psql

Change-Id: I67145808e8c453e371fe20575c5cc6a729db139a
This commit is contained in:
Ilya Kharin 2016-06-09 19:12:41 +03:00
parent 07aefb822d
commit cc6f6da8b8
9 changed files with 140 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
octane/handlers/backup_restore/__init__.py
View File

@ -19,7 +19,7 @@ from octane.handlers.backup_restore import fuel_uuid
# from octane.handlers.backup_restore import logs
from octane.handlers.backup_restore import mirrors
from octane.handlers.backup_restore import nailgun_plugins
# from octane.handlers.backup_restore import postgres
from octane.handlers.backup_restore import postgres
from octane.handlers.backup_restore import puppet
# from octane.handlers.backup_restore import release
from octane.handlers.backup_restore import ssh
@ -39,9 +39,8 @@ ARCHIVATORS = [
fuel_keys.FuelKeysArchivator,
fuel_uuid.FuelUUIDArchivator,
puppet.PuppetArchivator,
# postgres.KeystoneArchivator,
# Nailgun restore should be after puppet restore
# postgres.NailgunArchivator,
# Restore of Nailgun DB should go after restore of Puppet modules.
postgres.DatabasesArchivator,
# release.ReleaseArchivator,
# logs.LogsArchivator,
version.VersionArchivator,

6
octane/handlers/backup_restore/base.py
View File

@ -123,18 +123,14 @@ class PathFilterArchivator(Base):
class CmdArchivator(Base):
container = None
cmd = None
filename = None
def backup(self):
assert self.cmd
assert self.container
assert self.filename
archivate.archivate_container_cmd_output(
self.archive, self.container, self.cmd, self.filename)
archivate.archivate_cmd_output(self.archive, self.cmd, self.filename)
class DirsArchivator(Base):

64
octane/handlers/backup_restore/postgres.py
View File

@ -16,16 +16,15 @@ import six
from octane.handlers.backup_restore import base
from octane import magic_consts
from octane.util import docker
from octane.util import patch
from octane.util import puppet
from octane.util import sql
from octane.util import subprocess
class PostgresArchivatorMeta(type):
def __init__(cls, name, bases, attr):
super(PostgresArchivatorMeta, cls).__init__(name, bases, attr)
cls.container = "postgres"
if cls.db is not None and cls.cmd is None:
cls.cmd = ["sudo", "-u", "postgres", "pg_dump", "-C", cls.db]
if cls.db is not None and cls.filename is None:
@ -35,44 +34,43 @@ class PostgresArchivatorMeta(type):
@six.add_metaclass(PostgresArchivatorMeta)
class PostgresArchivator(base.CmdArchivator):
db = None
services = []
def restore(self):
dump = self.archive.extractfile(self.filename)
subprocess.call([
"systemctl", "stop", "docker-{0}.service".format(self.db)
])
docker.stop_container(self.db)
docker.run_in_container(
"postgres",
["sudo", "-u", "postgres", "dropdb", "--if-exists", self.db],
)
with docker.in_container("postgres",
["sudo", "-u", "postgres", "psql"],
stdin=subprocess.PIPE) as process:
subprocess.call(["systemctl", "stop"] + self.services)
subprocess.call(["sudo", "-u", "postgres", "dropdb", "--if-exists",
self.db])
with subprocess.popen(["sudo", "-u", "postgres", "psql"],
stdin=subprocess.PIPE) as process:
shutil.copyfileobj(dump, process.stdin)
docker.start_container(self.db)
docker.wait_for_container(self.db)
subprocess.call([
"systemctl", "start", "docker-{0}.service".format(self.db)
])
puppet.apply_task(self.db)
class NailgunArchivator(PostgresArchivator):
db = "nailgun"
services = [
"nailgun",
"oswl_flavor_collectord",
"oswl_image_collectord",
"oswl_keystone_user_collectord",
"oswl_tenant_collectord",
"oswl_vm_collectord",
"oswl_volume_collectord",
"receiverd",
"statsenderd",
"assassind",
]
patches = magic_consts.NAILGUN_ARCHIVATOR_PATCHES
def restore(self):
for args in magic_consts.NAILGUN_ARCHIVATOR_PATCHES:
docker.apply_patches(*args)
try:
with patch.applied_patch(*self.patches):
super(NailgunArchivator, self).restore()
self._repair_database_consistency()
finally:
for args in magic_consts.NAILGUN_ARCHIVATOR_PATCHES:
docker.apply_patches(*args, revert=True)
def _repair_database_consistency(self):
values = []
for line in sql.run_psql_in_container(
for line in sql.run_psql(
"select id, generated from attributes;", self.db):
c_id, c_data = line.split("|", 1)
data = json.loads(c_data)
@ -80,7 +78,7 @@ class NailgunArchivator(PostgresArchivator):
values.append("({0}, '{1}')".format(c_id, json.dumps(data)))
if values:
sql.run_psql_in_container(
sql.run_psql(
'update attributes as a set generated = b.generated '
'from (values {0}) as b(id, generated) '
'where a.id = b.id;'.format(','.join(values)),
@ -89,3 +87,15 @@ class NailgunArchivator(PostgresArchivator):
class KeystoneArchivator(PostgresArchivator):
db = "keystone"
services = ["openstack-keystone"]
class DatabasesArchivator(base.CollectionArchivator):
archivators_classes = [
KeystoneArchivator,
NailgunArchivator,
]
def restore(self):
puppet.apply_task("postgresql")
super(DatabasesArchivator, self).restore()

11
octane/magic_consts.py
View File

@ -18,13 +18,10 @@ CWD = os.path.dirname(__file__) # FIXME
FUEL_CACHE = "/tmp" # TODO: we shouldn't need this
PUPPET_DIR = "/etc/puppet/modules"
NAILGUN_ARCHIVATOR_PATCHES = [
(
"nailgun",
os.path.join(PUPPET_DIR, "nailgun/manifests/"),
os.path.join(CWD, "patches/timeout.patch")
),
]
NAILGUN_ARCHIVATOR_PATCHES = (
PUPPET_DIR,
os.path.join(CWD, "patches/timeout.patch"),
)
BOOTSTRAP_INITRAMFS = "/var/www/nailgun/bootstrap/initramfs.img"
SSH_KEYS = ['/root/.ssh/id_rsa', '/root/.ssh/bootstrap.rsa']

24
octane/patches/timeout.patch
View File

@ -1,12 +1,12 @@
diff --git a/deployment/puppet/nailgun/manifests/venv.pp b/deployment/puppet/nailgun/manifests/venv.pp
index 3313333..c383160 100644
--- venv.pp
+++ venv.pp
@@ -96,6 +96,7 @@ class nailgun::venv(
],
tries => 50,
try_sleep => 5,
+ timeout => 0,
}
exec {"nailgun_upload_fixtures":
command => "${venv}/bin/nailgun_fixtures",
diff --git a/fuel/manifests/nailgun/server.pp b/fuel/manifests/nailgun/server.pp
index a7624b0..1c9c831 100644
--- a/fuel/manifests/nailgun/server.pp
+++ b/fuel/manifests/nailgun/server.pp
@@ -116,6 +116,7 @@ class fuel::nailgun::server (
subscribe => File["/etc/nailgun/settings.yaml"],
tries => 50,
try_sleep => 5,
+ timeout => 0,
}
exec {"nailgun_upload_fixtures":

3
octane/tests/test_archivators.py
View File

@ -108,11 +108,10 @@ def test_path_filter_backup(mocker, cls, banned_files, backup_directory,
def test_posgres_archivator(mocker, cls, db):
test_archive = mocker.Mock()
archive_mock = mocker.patch(
"octane.util.archivate.archivate_container_cmd_output")
"octane.util.archivate.archivate_cmd_output")
cls(test_archive).backup()
archive_mock.assert_called_once_with(
test_archive,
"postgres",
["sudo", "-u", "postgres", "pg_dump", "-C", db],
"postgres/{0}.sql".format(db))

128
octane/tests/test_archivators_restore.py
View File

@ -210,90 +210,94 @@ def test_cobbler_archivator(mocker, mock_subprocess):
mock_puppet.assert_called_once_with("cobbler")
@pytest.mark.parametrize("cls,db,sync_db_cmd,mocked_actions_names", [
def test_databases_archivator(mocker):
mock_call = mock.Mock()
mocker.patch.object(postgres.NailgunArchivator, "restore",
new=mock_call.nailgun.restore)
mocker.patch.object(postgres.KeystoneArchivator, "restore",
new=mock_call.keystone.restore)
mocker.patch("octane.util.puppet.apply_task",
new=mock_call.puppet.apply_task)
archivator = postgres.DatabasesArchivator(mock.Mock(), mock.Mock())
archivator.restore()
assert mock_call.mock_calls == [
mock.call.puppet.apply_task("postgresql"),
mock.call.keystone.restore(),
mock.call.nailgun.restore(),
]
@pytest.mark.parametrize("cls,db,services,mocked_actions_names", [
(
postgres.NailgunArchivator,
"nailgun",
["nailgun_syncdb"],
[
"nailgun",
"oswl_flavor_collectord",
"oswl_image_collectord",
"oswl_keystone_user_collectord",
"oswl_tenant_collectord",
"oswl_vm_collectord",
"oswl_volume_collectord",
"receiverd",
"statsenderd",
"assassind",
],
["_repair_database_consistency"],
),
(
postgres.KeystoneArchivator,
"keystone",
["keystone-manage", "db_sync"],
["openstack-keystone"],
[],
),
])
def test_postgres_restore(mocker, cls, db, sync_db_cmd, mocked_actions_names):
patch_mock = mocker.patch("octane.util.docker.apply_patches")
def test_postgres_restore(mocker, cls, db, services, mocked_actions_names):
mocked_actions = []
for mocked_action_name in mocked_actions_names:
mocked_actions.append(mocker.patch.object(cls, mocked_action_name))
member = TestMember("postgres/{0}.sql".format(db), True, True)
archive = TestArchive([member], cls)
actions = []
def foo(action):
return_mock_object = mocker.Mock()
mock_subprocess = mock.MagicMock()
mocker.patch("octane.util.subprocess.call", new=mock_subprocess.call)
mocker.patch("octane.util.subprocess.popen", new=mock_subprocess.popen)
def mock_foo(*args, **kwargs):
actions.append(action)
return return_mock_object
mock_foo.return_value = return_mock_object
return mock_foo
mock_patch = mocker.patch("octane.util.patch.applied_patch")
mock_copyfileobj = mocker.patch("shutil.copyfileobj")
mock_apply_task = mocker.patch("octane.util.puppet.apply_task")
call_mock = mocker.patch("octane.util.subprocess.call",
side_effect=foo("call"))
in_container_mock = mocker.patch("octane.util.docker.in_container")
side_effect_in_container = foo("in_container")
in_container_mock.return_value.__enter__.side_effect = \
side_effect_in_container
run_in_container = mocker.patch(
"octane.util.docker.run_in_container",
side_effect=foo("run_in_container"))
mocker.patch("octane.util.docker.stop_container",
side_effect=foo("stop_container"))
mocker.patch("octane.util.docker.start_container",
side_effect=foo("start_container"))
mocker.patch("octane.util.docker.wait_for_container",
side_effect=foo("wait_for_container"))
cls(archive).restore()
member.assert_extract()
args = ["call", "stop_container", "run_in_container", "in_container",
"start_container", "wait_for_container", "call"]
assert args == actions
if cls is postgres.NailgunArchivator:
assert [
mock.call(
'nailgun',
'/etc/puppet/modules/nailgun/manifests/',
os.path.join(magic_consts.CWD, "patches/timeout.patch")
),
mock.call(
'nailgun',
'/etc/puppet/modules/nailgun/manifests/',
os.path.join(magic_consts.CWD, "patches/timeout.patch"),
revert=True
),
] == patch_mock.call_args_list
else:
assert not patch_mock.called
call_mock.assert_has_calls([
mock.call(["systemctl", "stop", "docker-{0}.service".format(db)]),
mock.call(["systemctl", "start", "docker-{0}.service".format(db)])
])
in_container_mock.assert_called_once_with(
"postgres",
["sudo", "-u", "postgres", "psql"],
stdin=subprocess.PIPE
assert mock_subprocess.mock_calls == [
mock.call.call(["systemctl", "stop"] + services),
mock.call.call(["sudo", "-u", "postgres", "dropdb", "--if-exists",
db]),
mock.call.popen(["sudo", "-u", "postgres", "psql"],
stdin=subprocess.PIPE),
mock.call.popen().__enter__(),
mock.call.popen().__exit__(None, None, None),
]
mock_copyfileobj.assert_called_once_with(
member,
mock_subprocess.popen.return_value.__enter__.return_value.stdin,
)
run_in_container.assert_has_calls([
mock.call("postgres",
["sudo", "-u", "postgres", "dropdb", "--if-exists", db]),
])
side_effect_in_container.return_value.stdin.write.assert_called_once_with(
member.dump)
mock_apply_task.assert_called_once_with(db)
if cls is postgres.NailgunArchivator:
assert mock_patch.call_args_list == [
mock.call(
'/etc/puppet/modules',
os.path.join(magic_consts.CWD, "patches/timeout.patch"),
),
]
else:
assert not mock_patch.called
for mocked_action in mocked_actions:
mocked_action.assert_called_once_with()
@ -526,7 +530,7 @@ def test_release_restore(mocker, mock_open, dump, calls):
def test_repair_database_consistency(mocker, mock_open):
run_sql_mock = mocker.patch(
"octane.util.sql.run_psql_in_container",
"octane.util.sql.run_psql",
return_value=["1|{}"],
)
json_mock = mocker.patch("json.dumps")

9
octane/util/archivate.py
View File

@ -61,6 +61,15 @@ def archivate_container_cmd_output(archive, container, cmd, filename):
archive.addfile(info, dump)
def archivate_cmd_output(archive, cmd, filename):
suffix = ".{0}".format(os.path.basename(filename))
with tempfile.NamedTemporaryFile(suffix=suffix) as f:
with subprocess.popen(cmd, stdout=subprocess.PIPE) as process:
shutil.copyfileobj(process.stdout, f)
f.flush()
archive.add(f.name, filename)
def filter_members(archive, dir_name):
if '/' not in dir_name:
dir_name = "{0}/".format(dir_name)

7
octane/util/sql.py
View File

@ -30,3 +30,10 @@ def run_psql_in_container(sql, db):
],
stdout=subprocess.PIPE)
return results.strip().splitlines()
def run_psql(sql, db):
output = subprocess.call_output(
["sudo", "-u", "postgres", "psql", db, "--tuples-only", "--no-align",
"-c", sql])
return output.strip().splitlines()