openstack/python-tripleoclient
Code Issues Proposed changes

Merge "Introduce "openstack overcloud ceph spec""

Browse Source
This commit is contained in:
Zuul 2022-02-02 01:44:11 +00:00 committed by Gerrit Code Review
commit 2e2dea300a
7 changed files with 382 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
releasenotes/notes/overcloud_ceph_spec-e1cfd358c4db2b22.yaml Normal file
View File

@ -0,0 +1,14 @@
---
features:
- |
New command "openstack overcloud ceph spec" has been added. This command
may be used to create a cephadm spec file as a function of the output of
metalsmith and a TripleO roles file. For example, if metalsmith output a
file with multiple hosts of differing roles and each role contained various
Ceph services, then a cephadm spec file could parse these files and return
input compatible with cephadm. The ceph spec file may be then be passed to
"openstack overcloud ceph deploy" so that cephadm deploys only those Ceph
services on those hosts. This feature should save users from the need to
create two different files containing much of the same data and make it
easier and less error prone to include Ceph in a deployment without the
need to manually create the Ceph spec file.

2
requirements.txt
View File

@ -11,7 +11,7 @@ python-ironicclient!=2.5.2,!=2.7.1,!=3.0.0,>=2.3.0 # Apache-2.0
python-openstackclient>=5.2.0 # Apache-2.0
simplejson>=3.5.1 # MIT
osc-lib>=2.3.0 # Apache-2.0
tripleo-common>=16.0.0 # Apache-2.0
tripleo-common>=16.3.0 # Apache-2.0
cryptography>=2.1 # BSD/Apache-2.0
ansible-runner>=1.4.5 # Apache 2.0
validations-libs>=1.5.0 # Apache-2.0

1
setup.cfg
View File

@ -43,6 +43,7 @@ openstack.tripleoclient.v2 =
overcloud_netenv_validate = tripleoclient.v1.overcloud_netenv_validate:ValidateOvercloudNetenv
overcloud_cell_export = tripleoclient.v1.overcloud_cell:ExportCell
overcloud_ceph_deploy = tripleoclient.v2.overcloud_ceph:OvercloudCephDeploy
overcloud_ceph_spec = tripleoclient.v2.overcloud_ceph:OvercloudCephSpec
overcloud_ceph_user_disable = tripleoclient.v2.overcloud_ceph:OvercloudCephUserDisable
overcloud_ceph_user_enable = tripleoclient.v2.overcloud_ceph:OvercloudCephUserEnable
overcloud_config_download = tripleoclient.v1.overcloud_config:DownloadConfig

62
tripleoclient/tests/test_utils.py
View File

@ -18,6 +18,7 @@ import ansible_runner
import argparse
import datetime
import fixtures
import io
import logging
import openstack
import os
@ -2713,3 +2714,64 @@ class TestGetHostsFromCephSpec(TestCase):
cfgfile.close()
self.assertEqual(expected, hosts)
class TestCephSpecStandalone(TestCase):
def test_ceph_spec_standalone(self):
hostname = utils.get_hostname()
expected = []
expected.append(yaml.safe_load('''
addr: 192.168.122.252
hostname: %s
labels:
- mon
- _admin
- osd
- mgr
service_type: host
''' % hostname))
expected.append(yaml.safe_load('''
placement:
hosts:
- %s
service_id: mon
service_name: mon
service_type: mon
''' % hostname))
expected.append(yaml.safe_load('''
placement:
hosts:
- %s
service_id: mgr
service_name: mgr
service_type: mgr
''' % hostname))
expected.append(yaml.safe_load('''
data_devices:
all: true
placement:
hosts:
- %s
service_id: default_drive_group
service_name: osd.default_drive_group
service_type: osd
''' % hostname))
expected_spec = tempfile.NamedTemporaryFile()
for spec in expected:
with open(expected_spec.name, 'a') as f:
f.write('---\n')
f.write(yaml.safe_dump(spec))
my_spec = tempfile.NamedTemporaryFile()
utils.ceph_spec_standalone(my_spec.name,
mon_ip='192.168.122.252')
self.assertCountEqual(
list(io.open(expected_spec.name)),
list(io.open(my_spec.name)))
expected_spec.close()
my_spec.close()

42
tripleoclient/tests/v2/overcloud_ceph/test_overcloud_ceph.py
View File

@ -299,3 +299,45 @@ class TestOvercloudCephUserEnable(fakes.FakePlaybookExecution):
"tripleo_cephadm_action": 'enable'
}
)
class TestOvercloudCephSpec(fakes.FakePlaybookExecution):
def setUp(self):
super(TestOvercloudCephSpec, self).setUp()
# Get the command object to test
app_args = mock.Mock()
app_args.verbose_level = 1
self.app.options = fakes.FakeOptions()
self.cmd = overcloud_ceph.OvercloudCephSpec(self.app,
app_args)
@mock.patch('tripleoclient.utils.TempDirs', autospect=True)
@mock.patch('os.path.abspath', autospect=True)
@mock.patch('os.path.exists', autospect=True)
@mock.patch('tripleoclient.utils.run_ansible_playbook', autospec=True)
def test_overcloud_ceph_spec(self, mock_playbook, mock_abspath,
mock_path_exists, mock_tempdirs):
arglist = ['deployed-metal.yaml', '--yes',
'--stack', 'overcloud',
'--roles-data', 'roles_data.yaml',
'--osd-spec', 'osd_spec.yaml',
'--output', 'ceph_spec.yaml']
parsed_args = self.check_parser(self.cmd, arglist, [])
self.cmd.take_action(parsed_args)
mock_playbook.assert_called_once_with(
playbook='cli-deployed-ceph.yaml',
inventory=mock.ANY,
workdir=mock.ANY,
playbook_dir=mock.ANY,
verbosity=3,
tags='ceph_spec',
reproduce_command=False,
extra_vars={
"baremetal_deployed_path": mock.ANY,
'tripleo_roles_path': mock.ANY,
'osd_spec_path': mock.ANY,
'ceph_spec_path': mock.ANY,
}
)

74
tripleoclient/utils.py
View File

@ -66,6 +66,7 @@ from tenacity.stop import stop_after_attempt, stop_after_delay
from tenacity.wait import wait_fixed
from tripleo_common.image import kolla_builder
from tripleo_common.utils import ceph_spec
from tripleo_common.utils import plan as plan_utils
from tripleo_common.utils import heat as tc_heat_utils
from tripleo_common.utils import stack as stack_utils
@ -2075,14 +2076,27 @@ def prepend_environment(environment_files, templates_dir, environment):
return environment_files
def get_hostname(short=False):
"""Returns the local hostname
:param (short): boolean true to run 'hostname -s'
:return string
"""
if short:
cmd = ["hostname", "-s"]
else:
cmd = ["hostname"]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
universal_newlines=True)
return p.communicate()[0].rstrip().lower()
def get_short_hostname():
"""Returns the local short hostname
:return string
"""
p = subprocess.Popen(["hostname", "-s"], stdout=subprocess.PIPE,
universal_newlines=True)
return p.communicate()[0].rstrip().lower()
return get_hostname(short=True)
def wait_api_port_ready(api_port, host='127.0.0.1'):
@ -3362,3 +3376,57 @@ def get_host_groups_from_ceph_spec(ceph_spec_path, prefix='',
"yaml.safe_load_all(%s) returned '%s'" % (ceph_spec_path, exc))
return hosts
def ceph_spec_standalone(ceph_spec_path, mon_ip, osd_spec_path=None):
"""Write ceph_spec_path file for a standalone ceph host
:param ceph_spec_path: the path to a ceph_spec.yaml file
:param mon_ip: the ip address of the ceph monitor
:param (osd_spec_path): path to an OSD spec file
:return None (writes file)
"""
specs = []
labels = ['osd', '_admin', 'mon', 'mgr']
host = get_hostname()
if osd_spec_path:
with open(os.path.abspath(osd_spec_path), 'r') as f:
try:
osd_spec = yaml.safe_load(f)
except yaml.YAMLError as exc:
raise oscexc.CommandError(
"Unable to parse '%s': %s"
% (os.path.abspath(osd_spec_path), exc))
else:
osd_spec = {
'data_devices': {
'all': True
}
}
placement_pattern = ''
spec_dict = {}
# create host spec
spec = ceph_spec.CephHostSpec('host', mon_ip, host, labels)
specs.append(spec.make_daemon_spec())
# add mon and mgr daemon specs
for svc in ['mon', 'mgr']:
d = ceph_spec.CephDaemonSpec(svc, svc, svc, [host],
placement_pattern, None,
spec_dict, labels)
specs.append(d.make_daemon_spec())
# add osd daemon spec
d = ceph_spec.CephDaemonSpec('osd', 'default_drive_group',
'osd.default_drive_group',
[host], placement_pattern,
None, spec_dict, labels,
**osd_spec)
specs.append(d.make_daemon_spec())
# render
open(ceph_spec_path, 'w').close() # reset file
for spec in specs:
with open(ceph_spec_path, 'a') as f:
f.write('---\n')
f.write(yaml.dump(spec))

191
tripleoclient/v2/overcloud_ceph.py
View File

@ -661,3 +661,194 @@ class OvercloudCephUserEnable(command.Command):
limit_hosts=ceph_hosts['_admin'][0],
reproduce_command=False,
)
class OvercloudCephSpec(command.Command):
log = logging.getLogger(__name__ + ".OvercloudCephSpec")
auth_required = False
def get_parser(self, prog_name):
parser = super(OvercloudCephSpec, self).get_parser(prog_name)
parser.add_argument('baremetal_env', nargs='?',
metavar='<deployed_baremetal.yaml>',
help=_('Path to the environment file '
'output from "openstack '
'overcloud node provision". '
'This argument may be excluded '
'only if --standalone is used.'))
parser.add_argument('-o', '--output', required=True,
metavar='<ceph_spec.yaml>',
help=_('The path to the output cephadm spec '
'file to pass to the "openstack '
'overcloud ceph deploy --ceph-spec '
'<ceph_spec.yaml>" command.'))
parser.add_argument('-y', '--yes', default=False, action='store_true',
help=_('Skip yes/no prompt before overwriting an '
'existing <ceph_spec.yaml> output file '
'(assume yes).'))
parser.add_argument('--stack', dest='stack',
help=_('Name or ID of heat stack '
'(default=Env: OVERCLOUD_STACK_NAME)'),
default=utils.env('OVERCLOUD_STACK_NAME',
default='overcloud'))
parser.add_argument(
'--working-dir', action='store',
help=_('The working directory for the deployment where all '
'input, output, and generated files will be stored.\n'
'Defaults to "$HOME/overcloud-deploy/<stack>"'))
parser.add_argument('--roles-data',
help=_(
"Path to an alternative roles_data.yaml. "
"Used to decide which node gets which "
"Ceph mon, mgr, or osd service "
"based on the node's role in "
"<deployed_baremetal.yaml>."),
default=os.path.join(
constants.TRIPLEO_HEAT_TEMPLATES,
constants.OVERCLOUD_ROLES_FILE))
parser.add_argument('--mon-ip',
help=_(
"IP address of the first Ceph monitor. "
"Only available with --standalone."),
default='')
parser.add_argument('--standalone', default=False,
action='store_true',
help=_("Create a spec file for a standalone "
"deployment. Used for single server "
"development or testing environments."))
spec_group = parser.add_mutually_exclusive_group()
spec_group.add_argument('--osd-spec',
help=_(
"Path to an existing OSD spec file. "
"When the Ceph spec file is generated "
"its OSD spec defaults to "
"{data_devices: {all: true}} "
"for all service_type osd. "
"Use --osd-spec to override the "
"data_devices value inside the "
"Ceph spec file."),
default=None)
spec_group.add_argument('--crush-hierarchy',
help=_(
"Path to an existing crush hierarchy spec "
"file. "),
default=None)
return parser
def take_action(self, parsed_args):
self.log.debug("take_action(%s)" % parsed_args)
output_path = os.path.abspath(parsed_args.output)
overwrite = parsed_args.yes
if (os.path.exists(output_path) and not overwrite
and not oooutils.prompt_user_for_confirmation(
'Overwrite existing file %s [y/N]?' % parsed_args.output,
self.log)):
raise oscexc.CommandError("Will not overwrite existing file:"
" %s. See the --yes parameter to "
"override this behavior. " %
parsed_args.output)
else:
overwrite = True
if not parsed_args.standalone:
if not parsed_args.working_dir:
working_dir = oooutils.get_default_working_dir(
parsed_args.stack)
else:
working_dir = os.path.abspath(parsed_args.working_dir)
oooutils.makedirs(working_dir)
inventory = os.path.join(working_dir,
constants.TRIPLEO_STATIC_INVENTORY)
if not os.path.exists(inventory):
raise oscexc.CommandError(
"Inventory file not found in working directory: "
"%s. It should have been created by "
"'openstack overcloud node provision'."
% inventory)
# mandatory extra_vars are now set, add others conditionally
extra_vars = {
'ceph_spec_path': output_path,
}
# optional paths to pass to playbook
if parsed_args.standalone is None and \
parsed_args.baremetal_env is None:
raise oscexc.CommandError(
"Either <deployed_baremetal.yaml> "
"or --standalone must be used.")
if parsed_args.baremetal_env:
baremetal_env_path = os.path.abspath(parsed_args.baremetal_env)
if not os.path.exists(baremetal_env_path):
raise oscexc.CommandError(
"Baremetal environment file does not exist:"
" %s" % parsed_args.baremetal_env)
else:
extra_vars['baremetal_deployed_path'] = \
os.path.abspath(parsed_args.baremetal_env)
if parsed_args.roles_data:
if not os.path.exists(parsed_args.roles_data):
raise oscexc.CommandError(
"Roles Data file not found --roles-data %s."
% os.path.abspath(parsed_args.roles_data))
else:
extra_vars['tripleo_roles_path'] = \
os.path.abspath(parsed_args.roles_data)
if parsed_args.mon_ip:
if not oooutils.is_valid_ip(parsed_args.mon_ip):
raise oscexc.CommandError(
"Invalid IP address '%s' passed to --mon-ip."
% parsed_args.mon_ip)
else:
if parsed_args.standalone:
extra_vars['tripleo_cephadm_first_mon_ip'] = \
parsed_args.mon_ip
else:
raise oscexc.CommandError(
"Option --mon-ip may only be "
"used with --standalone")
if parsed_args.osd_spec:
if not os.path.exists(parsed_args.osd_spec):
raise oscexc.CommandError(
"OSD Spec file not found --osd-spec %s."
% os.path.abspath(parsed_args.osd_spec))
else:
extra_vars['osd_spec_path'] = \
os.path.abspath(parsed_args.osd_spec)
if parsed_args.crush_hierarchy:
if not os.path.exists(parsed_args.crush_hierarchy):
raise oscexc.CommandError(
"Crush Hierarchy Spec file not found --crush-hierarchy %s."
% os.path.abspath(parsed_args.crush_hierarchy))
else:
extra_vars['crush_hierarchy_path'] = \
os.path.abspath(parsed_args.crush_hierarchy)
# Call the playbook to create the spec from baremetal and roles files
if not parsed_args.standalone:
with oooutils.TempDirs() as tmp:
oooutils.run_ansible_playbook(
playbook='cli-deployed-ceph.yaml',
inventory=inventory,
workdir=tmp,
playbook_dir=constants.ANSIBLE_TRIPLEO_PLAYBOOKS,
verbosity=oooutils.playbook_verbosity(self=self),
extra_vars=extra_vars,
reproduce_command=False,
tags='ceph_spec',
)
else:
# Create the spec directly
oooutils.ceph_spec_standalone(ceph_spec_path=output_path,
mon_ip=parsed_args.mon_ip,
osd_spec_path=parsed_args.osd_spec)