Merge "Graph-based upgrade approach. Change to upgrade_db."
This commit is contained in:
Jenkins
Gerrit Code Review
commit
de88d4b888
|
|
@ -20,6 +20,7 @@ from fuelclient.objects import environment as environment_obj
|
|||
|
||||
from octane import magic_consts
|
||||
from octane.util import db
|
||||
from octane.util import deployment as deploy
|
||||
from octane.util import env as env_util
|
||||
from octane.util import maintenance
|
||||
|
||||
|
|
@ -57,6 +58,24 @@ def upgrade_db(orig_id, seed_id, db_role_name):
|
|||
db.db_sync(seed_env)
|
||||
|
||||
|
||||
def upgrade_db_with_graph(orig_id, seed_id):
|
||||
"""Upgrade db using deployment graphs."""
|
||||
# Upload all graphs
|
||||
deploy.upload_graphs(orig_id, seed_id)
|
||||
|
||||
# If any failure try to rollback ONLY original environment.
|
||||
try:
|
||||
deploy.execute_graph_and_wait("upgrade-db-orig", orig_id)
|
||||
deploy.execute_graph_and_wait("upgrade-db-seed", seed_id)
|
||||
except Exception:
|
||||
cluster_graphs = deploy.get_cluster_graph_names(orig_id)
|
||||
if "upgrade-db-orig-rollback" in cluster_graphs:
|
||||
LOG.info("Trying to rollback 'upgrade-db' on the "
|
||||
"orig environment '%s'.", orig_id)
|
||||
deploy.execute_graph_and_wait("upgrade-db-orig-rollback", orig_id)
|
||||
raise
|
||||
|
||||
|
||||
class UpgradeDBCommand(cmd.Command):
|
||||
"""Migrate and upgrade state databases data"""
|
||||
|
||||
|
|
@ -69,13 +88,22 @@ class UpgradeDBCommand(cmd.Command):
|
|||
'seed_id', type=int, metavar='SEED_ID',
|
||||
help="ID of seed environment")
|
||||
|
||||
parser.add_argument(
|
||||
group = parser.add_mutually_exclusive_group()
|
||||
group.add_argument(
|
||||
'--db_role_name', type=str, metavar='DB_ROLE_NAME',
|
||||
default="controller", help="Set not standard role name for DB "
|
||||
"(default controller).")
|
||||
group.add_argument(
|
||||
'--with-graph', action='store_true',
|
||||
help="EXPERIMENTAL: Use Fuel deployment graphs"
|
||||
" instead of python-based commands.")
|
||||
|
||||
return parser
|
||||
|
||||
def take_action(self, parsed_args):
|
||||
upgrade_db(parsed_args.orig_id, parsed_args.seed_id,
|
||||
parsed_args.db_role_name)
|
||||
# Execute alternative approach if requested
|
||||
if parsed_args.with_graph:
|
||||
upgrade_db_with_graph(parsed_args.orig_id, parsed_args.seed_id)
|
||||
else:
|
||||
upgrade_db(parsed_args.orig_id, parsed_args.seed_id,
|
||||
parsed_args.db_role_name)
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ PATCHES_DIR = os.path.join(CWD, "patches")
|
|||
|
||||
FUEL_CACHE = "/tmp" # TODO: we shouldn't need this
|
||||
PUPPET_DIR = "/etc/puppet/modules"
|
||||
DEPLOYMENT_GRAPH_DIR = "/var/www/nailgun/octane/puppet/octane_tasks/graphs"
|
||||
|
||||
NAILGUN_ARCHIVATOR_PATCHES = (
|
||||
PUPPET_DIR,
|
||||
os.path.join(CWD, "patches/timeout.patch"),
|
||||
|
|
|
|||
|
|
@ -10,6 +10,11 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import mock
|
||||
import pytest
|
||||
|
||||
from octane.commands import upgrade_db
|
||||
|
||||
|
||||
def test_parser(mocker, octane_app):
|
||||
m = mocker.patch('octane.commands.upgrade_db.upgrade_db')
|
||||
|
|
@ -17,3 +22,119 @@ def test_parser(mocker, octane_app):
|
|||
assert not octane_app.stdout.getvalue()
|
||||
assert not octane_app.stderr.getvalue()
|
||||
m.assert_called_once_with(1, 2, '3')
|
||||
|
||||
|
||||
def test_parser_with_graph(mocker, octane_app):
|
||||
m = mocker.patch("octane.commands.upgrade_db.upgrade_db_with_graph")
|
||||
octane_app.run(["upgrade-db", "--with-graph", "1", "2"])
|
||||
assert not octane_app.stdout.getvalue()
|
||||
assert not octane_app.stderr.getvalue()
|
||||
m.assert_called_once_with(1, 2)
|
||||
|
||||
|
||||
def test_parser_exclusive_group(mocker, octane_app):
|
||||
mocker.patch("octane.commands.upgrade_db.upgrade_db")
|
||||
mocker.patch("octane.commands.upgrade_db.upgrade_db_with_graph")
|
||||
with pytest.raises(AssertionError):
|
||||
octane_app.run(["upgrade-db", "--with-graph", "--db_role_name", "db",
|
||||
"1", "2"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("calls", "graph_names", "catch"), [
|
||||
# Orig is fine, seed is fine and there is no need to rollback.
|
||||
(
|
||||
[
|
||||
("upgrade-db-orig", False),
|
||||
("upgrade-db-seed", False),
|
||||
],
|
||||
["upgrade-db-orig", "upgrade-db-orig-rollback", "upgrade-db-seed"],
|
||||
None,
|
||||
),
|
||||
# Orig is fine, seed fails and there is no rollback.
|
||||
(
|
||||
[
|
||||
("upgrade-db-orig", False),
|
||||
("upgrade-db-seed", True),
|
||||
],
|
||||
["upgrade-db-orig", "upgrade-db-seed"],
|
||||
"upgrade-db-seed",
|
||||
),
|
||||
# Orig is fine, seed fails and rollback is fine.
|
||||
(
|
||||
[
|
||||
("upgrade-db-orig", False),
|
||||
("upgrade-db-seed", True),
|
||||
("upgrade-db-orig-rollback", False),
|
||||
],
|
||||
["upgrade-db-orig", "upgrade-db-orig-rollback", "upgrade-db-seed"],
|
||||
"upgrade-db-seed",
|
||||
),
|
||||
# Orig is fine, seed fails and rollback fails too.
|
||||
(
|
||||
[
|
||||
("upgrade-db-orig", False),
|
||||
("upgrade-db-seed", True),
|
||||
("upgrade-db-orig-rollback", True),
|
||||
],
|
||||
["upgrade-db-orig", "upgrade-db-orig-rollback", "upgrade-db-seed"],
|
||||
"upgrade-db-orig-rollback",
|
||||
),
|
||||
# Orig fails and there is no rollback.
|
||||
(
|
||||
[
|
||||
("upgrade-db-orig", True),
|
||||
],
|
||||
["upgrade-db-orig", "upgrade-db-seed"],
|
||||
"upgrade-db-orig",
|
||||
),
|
||||
# Orig fails, rollback is fine.
|
||||
(
|
||||
[
|
||||
("upgrade-db-orig", True),
|
||||
("upgrade-db-orig-rollback", False),
|
||||
],
|
||||
["upgrade-db-orig", "upgrade-db-orig-rollback", "upgrade-db-seed"],
|
||||
"upgrade-db-orig",
|
||||
),
|
||||
# Orig fails, rollback is also fails.
|
||||
(
|
||||
[
|
||||
("upgrade-db-orig", True),
|
||||
("upgrade-db-orig-rollback", True),
|
||||
],
|
||||
["upgrade-db-orig", "upgrade-db-orig-rollback", "upgrade-db-seed"],
|
||||
"upgrade-db-orig-rollback",
|
||||
),
|
||||
])
|
||||
def test_upgrade_db_with_graph(mocker, calls, graph_names, catch):
|
||||
class ExecutionError(Exception):
|
||||
pass
|
||||
|
||||
def execute_graph(graph_name, env_id):
|
||||
assert graph_name in results, \
|
||||
"Unxpected execution of the graph {0}".format(graph_name)
|
||||
result = results[graph_name]
|
||||
if result is not None:
|
||||
raise result
|
||||
return mock.DEFAULT
|
||||
|
||||
results = {
|
||||
graph_name: ExecutionError(graph_name) if is_error else None
|
||||
for graph_name, is_error in calls
|
||||
}
|
||||
expected_exception = None
|
||||
if catch is not None:
|
||||
expected_exception = results[catch]
|
||||
|
||||
mocker.patch("octane.util.deployment.upload_graphs")
|
||||
mocker.patch("octane.util.deployment.execute_graph_and_wait",
|
||||
side_effect=execute_graph)
|
||||
mocker.patch("octane.util.deployment.get_cluster_graph_names",
|
||||
return_value=graph_names)
|
||||
|
||||
if expected_exception is not None:
|
||||
with pytest.raises(ExecutionError) as excinfo:
|
||||
upgrade_db.upgrade_db_with_graph(1, 2)
|
||||
assert excinfo.value is expected_exception
|
||||
else:
|
||||
upgrade_db.upgrade_db_with_graph(1, 2)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,129 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
|
||||
import mock
|
||||
import pytest
|
||||
|
||||
from octane.util import deployment
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("filename", "graph_name", "data", "is_error"), [
|
||||
("/a/b/c/orig/upgrade-db-org.yaml", "upgrade-db-org", {}, True),
|
||||
("/a/b/c/orig/upgrade-db-seed.yaml", "upgrade-db-seed", {}, True),
|
||||
("/a/b/c/seed/upgrade-db-orig.yaml", "upgrade-db-orig", {"a": "b"}, False),
|
||||
("/a/b/c/seed/upgrade-db-seed.yaml", "upgrade-db-seed", {"b": "c"}, False),
|
||||
])
|
||||
@pytest.mark.parametrize("env_id", [1, 2])
|
||||
def test_upload_graph_file_to_env(mocker, filename, graph_name, data, is_error,
|
||||
env_id):
|
||||
mock_load = mocker.patch("octane.util.helpers.load_yaml",
|
||||
return_value=data)
|
||||
mock_graph = mocker.patch("fuelclient.v1.graph.GraphClient")
|
||||
if is_error:
|
||||
with pytest.raises(Exception) as excinfo:
|
||||
deployment.upload_graph_file_to_env(filename, env_id)
|
||||
assert "Exception: Graph '{0}' is empty.".format(filename) == \
|
||||
excinfo.exconly()
|
||||
else:
|
||||
deployment.upload_graph_file_to_env(filename, env_id)
|
||||
mock_graph.return_value.upload.assert_called_once_with(
|
||||
data, "clusters", env_id, graph_name)
|
||||
mock_load.assert_called_once_with(filename)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("filenames", "expected"), [
|
||||
(["upgrade-db.yaml", "upgrade-db-seed.txt", "upgrade-db", "upgrade.yaml"],
|
||||
["upgrade-db.yaml", "upgrade.yaml"]),
|
||||
])
|
||||
@pytest.mark.parametrize("directory", ["/a/b/orig", "/a/b/seed"])
|
||||
@pytest.mark.parametrize("env_id", [1, 2])
|
||||
def test_upload_graphs_to_env(mocker, directory, filenames, expected,
|
||||
env_id):
|
||||
mock_listdir = mocker.patch("os.listdir", return_value=filenames)
|
||||
mock_upload = mocker.patch(
|
||||
"octane.util.deployment.upload_graph_file_to_env")
|
||||
deployment.upload_graphs_to_env(directory, env_id)
|
||||
assert mock_upload.call_args_list == [
|
||||
mock.call(os.path.join(directory, filename), env_id)
|
||||
for filename in expected
|
||||
]
|
||||
mock_listdir.assert_called_once_with(directory)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("orig_id", [1, 2])
|
||||
@pytest.mark.parametrize("seed_id", [3, 4])
|
||||
def test_upload_graphs(mocker, orig_id, seed_id):
|
||||
mock_upload = mocker.patch("octane.util.deployment.upload_graphs_to_env")
|
||||
deployment.upload_graphs(orig_id, seed_id)
|
||||
assert mock_upload.call_args_list == [
|
||||
mock.call("/var/www/nailgun/octane/puppet/octane_tasks/graphs/orig",
|
||||
orig_id),
|
||||
mock.call("/var/www/nailgun/octane/puppet/octane_tasks/graphs/seed",
|
||||
seed_id),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("statuses", "is_error", "is_timeout"), [
|
||||
(["pending", "running", "ready"], False, False),
|
||||
(["pending", "running"], False, True),
|
||||
(["pending", "pending"], False, True),
|
||||
(["pending", "running", "error"], True, False),
|
||||
])
|
||||
@pytest.mark.parametrize("graph_name", ["update-db-orig", "upgrade-db-seed"])
|
||||
@pytest.mark.parametrize("env_id", [1, 2])
|
||||
def test_execute_graph_and_wait(mocker, statuses, graph_name, env_id, is_error,
|
||||
is_timeout):
|
||||
def execute_graph():
|
||||
deployment.execute_graph_and_wait(graph_name, env_id,
|
||||
attempts=attempts)
|
||||
|
||||
mocker.patch("time.sleep")
|
||||
mock_status = mock.PropertyMock(side_effect=statuses)
|
||||
mock_task = mock.Mock(id=123)
|
||||
type(mock_task).status = mock_status
|
||||
mock_graph = mocker.patch("fuelclient.v1.graph.GraphClient")
|
||||
mock_graph.return_value.execute.return_value = mock_task
|
||||
|
||||
attempts = len(statuses)
|
||||
if is_error:
|
||||
with pytest.raises(Exception) as excinfo:
|
||||
execute_graph()
|
||||
assert excinfo.exconly().startswith(
|
||||
"Exception: Task 123 with graph {0}".format(graph_name))
|
||||
elif is_timeout:
|
||||
with pytest.raises(Exception) as excinfo:
|
||||
execute_graph()
|
||||
assert excinfo.exconly().startswith("Exception: Timeout waiting of")
|
||||
else:
|
||||
execute_graph()
|
||||
assert mock_status.call_count == attempts
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("graphs", "expected_names"), [
|
||||
(
|
||||
[
|
||||
{"relations": [{"type": "upgrade-orig"}]},
|
||||
{"relations": [{"type": "upgrade-seed"}]},
|
||||
],
|
||||
["upgrade-orig", "upgrade-seed"],
|
||||
),
|
||||
([], []),
|
||||
])
|
||||
@pytest.mark.parametrize("env_id", [1, 2])
|
||||
def test_get_cluster_graph_names(mocker, graphs, expected_names, env_id):
|
||||
mock_graph = mocker.patch("fuelclient.v1.graph.GraphClient")
|
||||
mock_graph.return_value.list.return_value = graphs
|
||||
names = deployment.get_cluster_graph_names(env_id)
|
||||
assert names == expected_names
|
||||
mock_graph.return_value.list.assert_called_once_with(env_id)
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging
|
||||
import os.path
|
||||
import time
|
||||
|
||||
from fuelclient.v1 import graph
|
||||
from octane import magic_consts
|
||||
from octane.util import helpers
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_cluster_graph_names(env_id):
|
||||
"""Return a list with graph names(types)."""
|
||||
|
||||
client = graph.GraphClient()
|
||||
# TODO: Take into account not only cluster graphs
|
||||
return [g['relations'][0]['type'] for g in client.list(env_id)]
|
||||
|
||||
|
||||
def upload_graphs(orig_id, seed_id):
|
||||
"""Upload upgrade graphs to Nailgun.
|
||||
|
||||
Read and upload graphs for original and seed environemtns
|
||||
from "orig" and "seed" subfolders respectevly.
|
||||
"""
|
||||
|
||||
# Upload command graphs to original environment
|
||||
orig_graph_dir = os.path.join(magic_consts.DEPLOYMENT_GRAPH_DIR, "orig")
|
||||
upload_graphs_to_env(orig_graph_dir, orig_id)
|
||||
|
||||
# Upload command graphs to seed environment
|
||||
seed_graph_dir = os.path.join(magic_consts.DEPLOYMENT_GRAPH_DIR, "seed")
|
||||
upload_graphs_to_env(seed_graph_dir, seed_id)
|
||||
|
||||
|
||||
def upload_graphs_to_env(directory, env_id):
|
||||
"""Upload all YAML-files as graphs to an environment."""
|
||||
|
||||
for filename in os.listdir(directory):
|
||||
if not filename.endswith(".yaml"):
|
||||
continue
|
||||
upload_graph_file_to_env(os.path.join(directory, filename), env_id)
|
||||
|
||||
|
||||
def upload_graph_file_to_env(graph_file_path, env_id):
|
||||
"""Upload a graph file to Nailgun for an environment."""
|
||||
|
||||
# Try to load graph data
|
||||
graph_data = helpers.load_yaml(graph_file_path)
|
||||
if not graph_data:
|
||||
raise Exception("Graph '{0}' is empty.".format(graph_file_path))
|
||||
graph_name = os.path.splitext(os.path.basename(graph_file_path))[0]
|
||||
|
||||
# Upload graph to Nailgun
|
||||
client = graph.GraphClient()
|
||||
client.upload(graph_data, "clusters", env_id, graph_name)
|
||||
LOG.info("Graph '%s' was uploaded for the environment '%s'.",
|
||||
graph_name, env_id)
|
||||
|
||||
|
||||
def execute_graph_and_wait(graph_name, env_id,
|
||||
attempts=120, attempt_delay=30):
|
||||
"""Execute graph with fuelclient and wait until finished."""
|
||||
|
||||
client = graph.GraphClient()
|
||||
graph_task = client.execute(env_id, None, graph_type=graph_name)
|
||||
for i in xrange(attempts):
|
||||
status = graph_task.status
|
||||
if status == 'ready':
|
||||
LOG.info("Graph %s for environment %s finished successfully.",
|
||||
graph_name, env_id)
|
||||
return
|
||||
elif status == 'error':
|
||||
raise Exception(
|
||||
"Task {0} with graph {1} for environment {2} finished with "
|
||||
"error.".format(graph_task.id, graph_name, env_id))
|
||||
LOG.info("Attempt %s: graph '%s' for environment %s has status %s",
|
||||
i, graph_name, env_id, status)
|
||||
time.sleep(attempt_delay)
|
||||
raise Exception("Timeout waiting of {0} seconds for the task {1} "
|
||||
"execution of the graph {2}."
|
||||
.format(attempts * attempt_delay, graph_task.id,
|
||||
graph_name))
|
||||
Loading…
Reference in New Issue