tripleo-common/scripts/tripleo-container-image-prepare
Alex Schultz 60afc0eec4 Make executor type dynamic
When we run the tripleo-container-image-prepare script, it performs
better under python2 when the process leverages a ProcessPoolExecutor.
Rather than using threading, we should be using processes to handle the
image upload processing. Currently when we're processing the images, we
end up being single threaded due to the GIL when processing the data. By
switching to the ProcessPoolExecutor, we eliminate the locking that is
occuring during the data processing as it'll be handled in each process.

Unfortunately, we cannot leverage the ProcessPoolExecutor when the same
code is run under Mistral. In order to make the code work for both
methods, we need to make the execution type dynamic. This change creates
two types of lock objects that are used to determine what type of
executor to ultimately use when processing the images for uploading.

Additionally this change limits the number of concurrent image upload
processes to 4 if using the ProcessPoolExecutor and caps the number of
threads at a max of 8 based on (cpu count / 2)

Change-Id: I60507eba9884a0660fe269da5ad27b0e57a70ca8
Related-Bug: #1844446
2019-10-08 08:25:21 -06:00

145 lines
4.7 KiB
Python
Executable File

#!/usr/bin/env python
# Copyright 2018 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import logging
import os
import sys
from tripleo_common import constants
from tripleo_common.image import image_uploader
from tripleo_common.image import kolla_builder
from tripleo_common.utils.locks import processlock
import yaml
def rel_or_abs_path(file_path, tht_root):
'''Find a file, either absolute path or relative to the t-h-t dir'''
if not file_path:
return None
path = os.path.abspath(file_path)
if not os.path.isfile(path):
path = os.path.abspath(os.path.join(tht_root, file_path))
if not os.path.isfile(path):
raise RuntimeError(
"Can't find path %s %s" % (file_path, path))
return path
def fetch_roles_file(roles_file, tht_path=constants.DEFAULT_TEMPLATES_PATH):
'''Fetch t-h-t roles data fromm roles_file abs path or rel to tht_path.'''
if not roles_file:
return None
with open(rel_or_abs_path(roles_file, tht_path)) as f:
return yaml.safe_load(f)
def get_args():
parser = argparse.ArgumentParser(
description=("tripleo-container-image-prepare"),
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
try:
roles_file = rel_or_abs_path(
constants.OVERCLOUD_J2_ROLES_NAME,
constants.DEFAULT_TEMPLATES_PATH)
except RuntimeError:
roles_file = None
parser.add_argument(
'--environment-file', '-e', metavar='<file path>', required=True,
help='Environment file containing the ContainerImagePrepare '
'parameter which specifies all prepare actions.'
)
parser.add_argument(
'--roles-file', '-r', dest='roles_file',
default=roles_file,
help='Roles file to filter images by, overrides the default %s'
% constants.OVERCLOUD_J2_ROLES_NAME
)
parser.add_argument(
"--cleanup",
dest="cleanup",
metavar='<full, partial, none>',
default=image_uploader.CLEANUP_FULL,
help="Cleanup behavior for local images left after upload. "
"The default 'full' will attempt to delete all local "
"images. 'partial' will leave images required for "
"deployment on this host. 'none' will do no cleanup."
)
parser.add_argument(
'--log-file', dest='log_file',
help='Log file to write prepare output to'
)
parser.add_argument(
'--dry-run',
dest='dry_run',
action='store_true',
default=False,
help='Do not perform any pull, modify, or push operations. '
'The environment file will still be populated as if these '
'operations were performed.'
)
parser.add_argument(
"--debug",
dest="debug",
action='store_true',
help="Enable debug logging. By default logging is set to INFO."
)
args = parser.parse_args(sys.argv[1:])
return args
if __name__ == '__main__':
args = get_args()
log_format = ('%(asctime)s %(process)d %(levelname)s '
'%(name)s [ ] %(message)s')
logging.basicConfig(
datefmt='%Y-%m-%d %H:%M:%S',
format=log_format
)
log = logging.getLogger()
if args.log_file:
formatter = logging.Formatter(log_format)
fh = logging.FileHandler(filename=args.log_file)
fh.setFormatter(formatter)
log.addHandler(fh)
if args.debug:
log_level = logging.DEBUG
else:
log_level = logging.INFO
log.setLevel(log_level)
if args.cleanup not in image_uploader.CLEANUP:
raise RuntimeError('--cleanup must be one of: %s' %
', '.join(image_uploader.CLEANUP))
roles_data = fetch_roles_file(args.roles_file)
with open(args.environment_file) as f:
env = yaml.safe_load(f)
try:
lock = processlock.ProcessLock()
params = kolla_builder.container_images_prepare_multi(
env, roles_data, cleanup=args.cleanup, dry_run=args.dry_run,
lock=lock)
result = yaml.safe_dump(params, default_flow_style=False)
log.info(result)
print(result)
except Exception as e:
log.exception("Image prepare failed: {}".format(e))
sys.exit(1)