Use Swift to store Plan environment

Start using the plan environment file in Swift for plan environment
storage instead of Mistral. Take care of migrating plan environment
data from Mistral to Swift when necessary.

Partially implements: blueprint stop-using-mistral-env
Co-Authored-By: Julie Pichon <jpichon@redhat.com>
Change-Id: I3bcef27413e685c498165b43a8b59c8c9cc5cf5e
This commit is contained in:
Ana Krivokapic 2017-04-05 21:21:25 +02:00 committed by Julie Pichon
parent 38b1a020f0
commit 961eca1397
5 changed files with 195 additions and 35 deletions

View File

@ -36,3 +36,6 @@ OVERCLOUD_YAML_NAME = "overcloud.yaml"
OVERCLOUD_ROLES_FILE = "roles_data.yaml"
RHEL_REGISTRATION_EXTRACONFIG_NAME = (
"extraconfig/pre_deploy/rhel-registration/")
# The name of the file which holds the plan environment contents
PLAN_ENVIRONMENT = 'plan-environment.yaml'

View File

@ -105,11 +105,14 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
mock_event.id = '1234'
mock_events.return_value = [mock_events]
workflow_client = clients.workflow_engine
workflow_client.environments.get.return_value = mock.MagicMock(
variables={'environments': []})
workflow_client.action_executions.create.return_value = mock.MagicMock(
output='{"result":[]}')
object_client = clients.tripleoclient.object_store
object_client.get_object = mock.Mock()
mock_env = yaml.safe_dump({'environments': []})
object_client.get_object.return_value = ({}, mock_env)
mock_check_hypervisor_stats.return_value = {
'count': 4,
'memory_mb': 4096,
@ -214,11 +217,14 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
mock_stack = fakes.create_tht_stack()
orchestration_client.stacks.get.side_effect = [None, mock.Mock()]
workflow_client = clients.workflow_engine
workflow_client.environments.get.return_value = mock.MagicMock(
variables={'environments': []})
workflow_client.action_executions.create.return_value = mock.MagicMock(
output='{"result":[]}')
object_client = clients.tripleoclient.object_store
object_client.get_object = mock.Mock()
mock_env = yaml.safe_dump({'environments': []})
object_client.get_object.return_value = ({}, mock_env)
def _orch_clt_create(**kwargs):
orchestration_client.stacks.get.return_value = mock_stack
@ -339,8 +345,6 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
mock_stack = fakes.create_tht_stack()
orchestration_client.stacks.get.side_effect = [None, mock.Mock()]
workflow_client = clients.workflow_engine
workflow_client.environments.get.return_value = mock.MagicMock(
variables={'environments': []})
workflow_client.action_executions.create.return_value = mock.MagicMock(
output='{"result":[]}')
@ -349,6 +353,11 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
orchestration_client.stacks.create.side_effect = _orch_clt_create
object_client = clients.tripleoclient.object_store
object_client.get_object = mock.Mock()
mock_env = yaml.safe_dump({'environments': []})
object_client.get_object.return_value = ({}, mock_env)
mock_check_hypervisor_stats.return_value = {
'count': 4,
'memory_mb': 4096,
@ -436,11 +445,14 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
baremetal.node.list.return_value = range(10)
workflow_client = clients.workflow_engine
workflow_client.environments.get.return_value = mock.MagicMock(
variables={'environments': []})
workflow_client.action_executions.create.return_value = mock.MagicMock(
output='{"result":[]}')
object_client = clients.tripleoclient.object_store
object_client.get_object = mock.Mock()
mock_env = yaml.safe_dump({'environments': []})
object_client.get_object.return_value = ({}, mock_env)
with mock.patch('tempfile.mkstemp') as mkstemp:
mkstemp.return_value = (os.open(self.parameter_defaults_env_file,
os.O_RDWR),
@ -767,11 +779,14 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
orchestration_client.stacks.get.return_value = fakes.create_tht_stack()
mock_events.return_value = []
workflow_client = clients.workflow_engine
workflow_client.environments.get.return_value = mock.MagicMock(
variables={'environments': []})
workflow_client.action_executions.create.return_value = mock.MagicMock(
output='{"result":[]}')
object_client = clients.tripleoclient.object_store
object_client.get_object = mock.Mock()
mock_env = yaml.safe_dump({'environments': []})
object_client.get_object.return_value = ({}, mock_env)
mock_check_hypervisor_stats.return_value = {
'count': 4,
'memory_mb': 4096,
@ -1037,11 +1052,14 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
clients = self.app.client_manager
workflow_client = clients.workflow_engine
workflow_client.environments.get.return_value = mock.MagicMock(
variables={'environments': []})
workflow_client.action_executions.create.return_value = mock.MagicMock(
output='{"result":[]}')
object_client = clients.tripleoclient.object_store
object_client.get_object = mock.Mock()
mock_env = yaml.safe_dump({'environments': []})
object_client.get_object.return_value = ({}, mock_env)
def _custom_create_params_env(parameters):
parameter_defaults = {"parameter_defaults": parameters}
return parameter_defaults
@ -1122,11 +1140,14 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
orchestration_client.stacks.create.side_effect = _orch_clt_create
workflow_client = clients.workflow_engine
workflow_client.environments.get.return_value = mock.MagicMock(
variables={'environments': []})
workflow_client.action_executions.create.return_value = mock.MagicMock(
output='{"result":[]}')
object_client = clients.tripleoclient.object_store
object_client.get_object = mock.Mock()
mock_env = yaml.safe_dump({'environments': []})
object_client.get_object.return_value = ({}, mock_env)
mock_check_hypervisor_stats.return_value = {
'count': 4,
'memory_mb': 4096,
@ -1208,11 +1229,14 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
]
workflow_client = clients.workflow_engine
workflow_client.environments.get.return_value = mock.MagicMock(
variables={'environments': []})
workflow_client.action_executions.create.return_value = mock.MagicMock(
output='{"result":[]}')
object_client = clients.tripleoclient.object_store
object_client.get_object = mock.Mock()
mock_env = yaml.safe_dump({'environments': []})
object_client.get_object.return_value = ({}, mock_env)
mock_relpath.return_value = './'
mock_get_template_contents.return_value = [{}, {}]

View File

@ -15,8 +15,10 @@
import mock
from osc_lib.tests import utils
from swiftclient import exceptions as swift_exc
from tripleoclient import exceptions
from tripleoclient.tests import base
from tripleoclient.workflows import plan_management
@ -152,3 +154,51 @@ class TestPlanCreationWorkflows(utils.TestCommand):
workflow_input={'queue_name': 'UUID4',
'container': 'test-overcloud',
'generate_passwords': False})
class TestUpdatePasswords(base.TestCase):
YAML_CONTENTS = """version: 1.0
name: overcloud
template: overcloud.yaml
parameter_defaults:
ControllerCount: 7
"""
def setUp(self):
super(TestUpdatePasswords, self).setUp()
self.swift_client = mock.MagicMock()
self.swift_client.get_object.return_value = ({}, self.YAML_CONTENTS)
self.plan_name = "overcast"
def test_update_passwords(self):
plan_management._update_passwords(self.swift_client,
self.plan_name,
{'AdminPassword': "1234"})
self.swift_client.put_object.assert_called_once()
result = self.swift_client.put_object.call_args_list[0][0][2]
# Check new data is in
self.assertIn("passwords:\n", result)
self.assertIn("\n AdminPassword: '1234'", result)
# Check previous data still is too
self.assertIn("name: overcloud", result)
def test_no_passwords(self):
plan_management._update_passwords(self.swift_client,
self.plan_name,
[])
self.swift_client.put_object.assert_not_called()
def test_no_plan_environment(self):
self.swift_client.get_object.side_effect = (
swift_exc.ClientException("404"))
plan_management._update_passwords(self.swift_client,
self.plan_name,
{'SecretPassword': 'abcd'})
self.swift_client.put_object.assert_not_called()

View File

@ -28,6 +28,8 @@ import yaml
from heatclient.common import template_utils
from heatclient import exc as hc_exc
from keystoneauth1 import exceptions as keystoneauth_exc
from mistralclient.api import base as mistralclient_base
from osc_lib.command import command
from osc_lib import exceptions as oscexc
from osc_lib.i18n import _
@ -284,20 +286,51 @@ class DeployOvercloud(command.Command):
contents = yaml.safe_dump(env)
# Until we have a well defined plan update workflow in tripleo-common
# we need to manually add an environment in swift and mistral for users
# we need to manually add an environment in swift and for users
# custom environments passed to the deploy command.
# See bug: https://bugs.launchpad.net/tripleo/+bug/1623431
# Update plan env while taking care to migrate it from Mistral to
# Swift.
swift_path = "user-environment.yaml"
self.object_client.put_object(container_name, swift_path, contents)
mistral_env = self.workflow_client.environments.get(container_name)
env_missing = env_changed = False
try:
env = yaml.safe_load(self.object_client.get_object(
container_name, constants.PLAN_ENVIRONMENT)[1])
except ClientException:
env_missing = True
env = self.workflow_client.environments.get(
container_name).variables
# TODO(akrivoka): delete env from Mistral once tripleo-common
# change merges (https://review.openstack.org/#/c/452291/)
else:
# If the plan environment exists, the Mistral environment
# is superseded and should be cleaned up.
try:
self.workflow_client.environments.delete(container_name)
except (mistralclient_base.APIException,
keystoneauth_exc.http.NotFound):
pass
user_env = {'path': swift_path}
if user_env not in mistral_env.variables['environments']:
mistral_env.variables['environments'].append(user_env)
self.workflow_client.environments.update(
name=container_name,
variables=mistral_env.variables
)
if user_env not in env['environments']:
env_changed = True
env['environments'].append(user_env)
if env_missing or env_changed:
yaml_string = yaml.safe_dump(env, default_flow_style=False)
self.object_client.put_object(
container_name, constants.PLAN_ENVIRONMENT, yaml_string)
# TODO(akrivoka): don't update env in Mistral once the
# tripleo-common change merges
# (https://review.openstack.org/#/c/452291/)
if env_missing:
self.workflow_client.environments.update(
name=container_name,
variables=env
)
def _upload_missing_files(self, container_name, files_dict, tht_root):
"""Find the files referenced in custom environments and upload them

View File

@ -9,9 +9,14 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import tempfile
import uuid
import yaml
from keystoneauth1 import exceptions as keystoneauth_exc
from mistralclient.api import base as mistralclient_base
from swiftclient import exceptions as swift_exc
from tripleo_common.utils import swift as swiftutils
from tripleo_common.utils import tarball
@ -19,7 +24,7 @@ from tripleoclient import constants
from tripleoclient import exceptions
from tripleoclient.workflows import base
LOG = logging.getLogger(__name__)
# Plan management workflows should generally be quick. However, the creation
# of the default plan in instack has demonstrated that sometimes it can take
# several minutes. This timeout value of 6 minutes is the same as the timeout
@ -164,32 +169,77 @@ def update_plan_from_templates(clients, name, tht_root, roles_file=None,
generate_passwords=True):
swift_client = clients.tripleoclient.object_store
# If the plan environment was migrated to Swift, save the generated
# 'passwords' if they exist as they can't be recreated from the
# templates content.
passwords = []
try:
env = yaml.safe_load(swift_client.get_object(
name, constants.PLAN_ENVIRONMENT)[1])
passwords = env.get('passwords', [])
except swift_exc.ClientException:
pass
# TODO(dmatthews): Removing the existing plan files should probably be
# a Mistral action.
print("Removing the current plan files")
swiftutils.empty_container(swift_client, name)
# Until we have a well defined plan update workflow in tripleo-common we
# need to manually reset the environments here. This is to ensure that
# no environments are in the mistral environment but not in swift.
# Until we have a well defined plan update workflow in
# tripleo-common we need to manually reset the environments and
# parameter_defaults here. This is to ensure that no environments
# are in the plan environment but not actually in swift.
# See bug: https://bugs.launchpad.net/tripleo/+bug/1623431
#
# Currently this is being done incidentally because we overwrite
# the existing plan-environment.yaml with the skeleton one in THT
# when updating the templates. Once LP#1623431 is resolved we may
# need to special-case plan-environment.yaml to avoid this.
# TODO(jpichon): Remove all these references to Mistral once
# https://review.openstack.org/#/c/452291/ merges.
mistral = clients.workflow_engine
mistral_env = mistral.environments.get(name)
mistral_env.variables['environments'] = []
mistral_env.variables['parameter_defaults'] = {}
mistral.environments.update(
name=name,
variables=mistral_env.variables
)
try:
mistral_env = mistral.environments.get(name)
except (mistralclient_base.APIException, keystoneauth_exc.http.NotFound):
# Plan was fully migrated, we can ignore.
pass
else:
mistral_env.variables['environments'] = []
mistral_env.variables['parameter_defaults'] = {}
mistral.environments.update(
name=name,
variables=mistral_env.variables
)
print("Uploading new plan files")
_upload_templates(swift_client, name, tht_root, roles_file)
_update_passwords(swift_client, name, passwords)
update_deployment_plan(clients, container=name,
queue_name=str(uuid.uuid4()),
generate_passwords=generate_passwords,
source_url=None)
def _update_passwords(swift_client, name, passwords):
# Update the plan environment with the generated passwords. This
# will be solved more elegantly once passwords are saved in a
# separate environment (https://review.openstack.org/#/c/467909/)
if passwords:
try:
env = yaml.safe_load(swift_client.get_object(
name, constants.PLAN_ENVIRONMENT)[1])
env['passwords'] = passwords
swift_client.put_object(name,
constants.PLAN_ENVIRONMENT,
yaml.safe_dump(env,
default_flow_style=False))
except swift_exc.ClientException:
# The plan likely has not been migrated to using Swift yet.
LOG.debug("Could not find plan environment %s in %s",
constants.PLAN_ENVIRONMENT, name)
def export_deployment_plan(clients, **workflow_input):
workflow_client = clients.workflow_engine
tripleoclients = clients.tripleoclient