diff --git a/etc/tempest.conf.sample b/etc/tempest.conf.sample index 8d0e97862a..f306d8e995 100644 --- a/etc/tempest.conf.sample +++ b/etc/tempest.conf.sample @@ -28,7 +28,7 @@ # format string to use for log messages with context (string # value) -#logging_context_format_string=%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user)s %(tenant)s] %(instance)s%(message)s +#logging_context_format_string=%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s # format string to use for log messages without context # (string value) @@ -43,7 +43,7 @@ #logging_exception_prefix=%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s # list of logger=LEVEL pairs (list value) -#default_log_levels=amqplib=WARN,sqlalchemy=WARN,boto=WARN,suds=INFO,keystone=INFO,paramiko=INFO +#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,iso8601=WARN # publish error events (boolean value) #publish_errors=false @@ -59,12 +59,13 @@ # it like this (string value) #instance_uuid_format="[instance: %(uuid)s] " -# If this option is specified, the logging configuration file -# specified is used and overrides any other logging options -# specified. Please see the Python logging module -# documentation for details on logging configuration files. -# (string value) -#log_config= +# The name of logging configuration file. It does not disable +# existing loggers, but just appends specified logging +# configuration to any other existing logging options. Please +# see the Python logging module documentation for details on +# logging configuration files. (string value) +# Deprecated group/name - [DEFAULT]/log_config +#log_config_append= # DEPRECATED. A logging.Formatter log message format string # which may use any of the available logging.LogRecord @@ -94,377 +95,67 @@ #syslog_log_facility=LOG_USER -[image] +[boto] # # Options defined in tempest.config # -# Catalog type of the Image service. (string value) -#catalog_type=image +# EC2 URL (string value) +#ec2_url=http://localhost:8773/services/Cloud -# The image region name to use. If empty, the value of -# identity.region is used instead. If no such region is found -# in the service catalog, the first found one is used. (string +# S3 URL (string value) +#s3_url=http://localhost:8080 + +# AWS Secret Key (string value) +#aws_secret= + +# AWS Access Key (string value) +#aws_access= + +# S3 Materials Path (string value) +#s3_materials_path=/opt/stack/devstack/files/images/s3-materials/cirros-0.3.0 + +# ARI Ramdisk Image manifest (string value) +#ari_manifest=cirros-0.3.0-x86_64-initrd.manifest.xml + +# AMI Machine Image manifest (string value) +#ami_manifest=cirros-0.3.0-x86_64-blank.img.manifest.xml + +# AKI Kernel Image manifest (string value) +#aki_manifest=cirros-0.3.0-x86_64-vmlinuz.manifest.xml + +# Instance type (string value) +#instance_type=m1.tiny + +# boto Http socket timeout (integer value) +#http_socket_timeout=3 + +# boto num_retries on error (integer value) +#num_retries=1 + +# Status Change Timeout (integer value) +#build_timeout=60 + +# Status Change Test Interval (integer value) +#build_interval=1 + + +[cli] + +# +# Options defined in tempest.cli +# + +# enable cli tests (boolean value) +#enabled=true + +# directory where python client binaries are located (string # value) -#region= +#cli_dir=/usr/local/bin -# http accessible image (string value) -#http_image=http://download.cirros-cloud.net/0.3.1/cirros-0.3.1-x86_64-uec.tar.gz - - -[object-storage] - -# -# Options defined in tempest.config -# - -# Catalog type of the Object-Storage service. (string value) -#catalog_type=object-store - -# The object-storage region name to use. If empty, the value -# of identity.region is used instead. If no such region is -# found in the service catalog, the first found one is used. -# (string value) -#region= - -# Number of seconds to time on waiting for a containerto -# container synchronization complete. (integer value) -#container_sync_timeout=120 - -# Number of seconds to wait while looping to check thestatus -# of a container to container synchronization (integer value) -#container_sync_interval=5 - -# Role to add to users created for swift tests to enable -# creating containers (string value) -#operator_role=Member - - -[volume-feature-enabled] - -# -# Options defined in tempest.config -# - -# Runs Cinder multi-backend test (requires 2 backends) -# (boolean value) -#multi_backend=false - -# A list of enabled extensions with a special entry all which -# indicates every extension is enabled (list value) -#api_extensions=all - -# Is the v1 volume API enabled (boolean value) -#api_v1=true - - -[telemetry] - -# -# Options defined in tempest.config -# - -# Catalog type of the Telemetry service. (string value) -#catalog_type=metering - - -[data_processing] - -# -# Options defined in tempest.config -# - -# Catalog type of the data processing service. (string value) -#catalog_type=data_processing - - -[object-storage-feature-enabled] - -# -# Options defined in tempest.config -# - -# Set to True if the Container Quota middleware is enabled -# (boolean value) -#container_quotas=true - -# Set to True if the Account Quota middleware is enabled -# (boolean value) -#accounts_quotas=true - -# Set to True if the Crossdomain middleware is enabled -# (boolean value) -#crossdomain=true - -# Set to True if the TempURL middleware is enabled (boolean -# value) -#tempurl=true - - -[network-feature-enabled] - -# -# Options defined in tempest.config -# - -# A list of enabled extensions with a special entry all which -# indicates every extension is enabled (list value) -#api_extensions=all - - -[image-feature-enabled] - -# -# Options defined in tempest.config -# - -# Is the v2 image API enabled (boolean value) -#api_v2=true - -# Is the v1 image API enabled (boolean value) -#api_v1=true - - -[compute-admin] - -# -# Options defined in tempest.config -# - -# Administrative Username to use for Nova API requests. -# (string value) -#username=admin - -# Administrative Tenant name to use for Nova API requests. -# (string value) -#tenant_name=admin - -# API key to use when authenticating as admin. (string value) -#password=pass - - -[volume] - -# -# Options defined in tempest.config -# - -# Time in seconds between volume availability checks. (integer -# value) -#build_interval=10 - -# Timeout in seconds to wait for a volume to becomeavailable. -# (integer value) -#build_timeout=300 - -# Catalog type of the Volume Service (string value) -#catalog_type=volume - -# The volume region name to use. If empty, the value of -# identity.region is used instead. If no such region is found -# in the service catalog, the first found one is used. (string -# value) -#region= - -# Name of the backend1 (must be declared in cinder.conf) -# (string value) -#backend1_name=BACKEND_1 - -# Name of the backend2 (must be declared in cinder.conf) -# (string value) -#backend2_name=BACKEND_2 - -# Backend protocol to target when creating volume types -# (string value) -#storage_protocol=iSCSI - -# Backend vendor to target when creating volume types (string -# value) -#vendor_name=Open Source - -# Disk format to use when copying a volume to image (string -# value) -#disk_format=raw - - -[compute-feature-enabled] - -# -# Options defined in tempest.config -# - -# If false, skip all nova v3 tests. (boolean value) -#api_v3=true - -# If false, skip disk config tests (boolean value) -#disk_config=true - -# A list of enabled extensions with a special entry all which -# indicates every extension is enabled (list value) -#api_extensions=all - -# A list of enabled v3 extensions with a special entry all -# which indicates every extension is enabled (list value) -#api_v3_extensions=all - -# Does the test environment support changing the admin -# password? (boolean value) -#change_password=false - -# Does the test environment support snapshots? (boolean value) -#create_image=false - -# Does the test environment support resizing? (boolean value) -#resize=false - -# Does the test environment support live migration available? -# (boolean value) -#live_migration=false - -# Does the test environment use block devices for live -# migration (boolean value) -#block_migration_for_live_migration=false - -# Does the test environment block migration support cinder -# iSCSI volumes (boolean value) -#block_migrate_cinder_iscsi=false - - -[identity] - -# -# Options defined in tempest.config -# - -# Catalog type of the Identity service. (string value) -#catalog_type=identity - -# Set to True if using self-signed SSL certificates. (boolean -# value) -#disable_ssl_certificate_validation=false - -# Full URI of the OpenStack Identity API (Keystone), v2 -# (string value) -#uri= - -# Full URI of the OpenStack Identity API (Keystone), v3 -# (string value) -#uri_v3= - -# The identity region name to use. Also used as the other -# services' region name unless they are set explicitly. If no -# such region is found in the service catalog, the first found -# one is used. (string value) -#region=RegionOne - -# Username to use for Nova API requests. (string value) -#username=demo - -# Tenant name to use for Nova API requests. (string value) -#tenant_name=demo - -# Role required to administrate keystone. (string value) -#admin_role=admin - -# API key to use when authenticating. (string value) -#password=pass - -# Username of alternate user to use for Nova API requests. -# (string value) -#alt_username= - -# Alternate user's Tenant name to use for Nova API requests. -# (string value) -#alt_tenant_name= - -# API key to use when authenticating as alternate user. -# (string value) -#alt_password= - -# Administrative Username to use forKeystone API requests. -# (string value) -#admin_username=admin - -# Administrative Tenant name to use for Keystone API requests. -# (string value) -#admin_tenant_name=admin - -# API key to use when authenticating as admin. (string value) -#admin_password=pass - - -[network] - -# -# Options defined in tempest.config -# - -# Catalog type of the Neutron service. (string value) -#catalog_type=network - -# The network region name to use. If empty, the value of -# identity.region is used instead. If no such region is found -# in the service catalog, the first found one is used. (string -# value) -#region= - -# The cidr block to allocate tenant networks from (string -# value) -#tenant_network_cidr=10.100.0.0/16 - -# The mask bits for tenant networks (integer value) -#tenant_network_mask_bits=28 - -# Whether tenant network connectivity should be evaluated -# directly (boolean value) -#tenant_networks_reachable=false - -# Id of the public network that provides external connectivity -# (string value) -#public_network_id= - -# Id of the public router that provides external connectivity -# (string value) -#public_router_id= - - -[stress] - -# -# Options defined in tempest.config -# - -# Directory containing log files on the compute nodes (string -# value) -#nova_logdir= - -# Maximum number of instances to create during test. (integer -# value) -#max_instances=16 - -# Controller host. (string value) -#controller= - -# Controller host. (string value) -#target_controller= - -# ssh user. (string value) -#target_ssh_user= - -# Path to private key. (string value) -#target_private_key_path= - -# regexp for list of log files. (string value) -#target_logfiles= - -# time (in seconds) between log file error checks. (integer -# value) -#log_check_interval=60 - -# The number of threads created while stress test. (integer -# value) -#default_thread_number_per_action=4 +# Number of seconds to wait on a CLI timeout (integer value) +#timeout=15 [compute] @@ -577,94 +268,68 @@ # (integer value) #shelved_offload_time=0 +# Allows test cases to create/destroy tenants and users. This +# option enables isolated test cases and better parallel +# execution, but also requires that OpenStack Identity API +# admin credentials are known. (boolean value) +#allow_tenant_isolation=false -[scenario] + +[compute-admin] # # Options defined in tempest.config # -# Directory containing image files (string value) -#img_dir=/opt/stack/new/devstack/files/images/cirros-0.3.1-x86_64-uec +# Administrative Username to use for Nova API requests. +# (string value) +#username=admin -# AMI image file name (string value) -#ami_img_file=cirros-0.3.1-x86_64-blank.img +# Administrative Tenant name to use for Nova API requests. +# (string value) +#tenant_name=admin -# ARI image file name (string value) -#ari_img_file=cirros-0.3.1-x86_64-initrd - -# AKI image file name (string value) -#aki_img_file=cirros-0.3.1-x86_64-vmlinuz - -# ssh username for the image file (string value) -#ssh_user=cirros - -# specifies how many resources to request at once. Used for -# large operations testing. (integer value) -#large_ops_number=0 +# API key to use when authenticating as admin. (string value) +#password=pass -[boto] +[compute-feature-enabled] # # Options defined in tempest.config # -# EC2 URL (string value) -#ec2_url=http://localhost:8773/services/Cloud +# If false, skip all nova v3 tests. (boolean value) +#api_v3=true -# S3 URL (string value) -#s3_url=http://localhost:8080 +# If false, skip disk config tests (boolean value) +#disk_config=true -# AWS Secret Key (string value) -#aws_secret= +# A list of enabled v3 extensions with a special entry all +# which indicates every extension is enabled (list value) +#api_v3_extensions=all -# AWS Access Key (string value) -#aws_access= +# Does the test environment support changing the admin +# password? (boolean value) +#change_password=false -# S3 Materials Path (string value) -#s3_materials_path=/opt/stack/devstack/files/images/s3-materials/cirros-0.3.0 +# Does the test environment support snapshots? (boolean value) +#create_image=false -# ARI Ramdisk Image manifest (string value) -#ari_manifest=cirros-0.3.0-x86_64-initrd.manifest.xml +# Does the test environment support resizing? (boolean value) +#resize=false -# AMI Machine Image manifest (string value) -#ami_manifest=cirros-0.3.0-x86_64-blank.img.manifest.xml +# Does the test environment support live migration available? +# (boolean value) +#live_migration=false -# AKI Kernel Image manifest (string value) -#aki_manifest=cirros-0.3.0-x86_64-vmlinuz.manifest.xml +# Does the test environment use block devices for live +# migration (boolean value) +#block_migration_for_live_migration=false -# Instance type (string value) -#instance_type=m1.tiny - -# boto Http socket timeout (integer value) -#http_socket_timeout=3 - -# boto num_retries on error (integer value) -#num_retries=1 - -# Status Change Timeout (integer value) -#build_timeout=60 - -# Status Change Test Interval (integer value) -#build_interval=1 - - -[cli] - -# -# Options defined in tempest.cli -# - -# enable cli tests (boolean value) -#enabled=true - -# directory where python client binaries are located (string -# value) -#cli_dir=/usr/local/bin - -# Number of seconds to wait on a CLI timeout (integer value) -#timeout=15 +# Does the test environment block migration support cinder +# iSCSI volumes (boolean value) +#block_migrate_cinder_iscsi=false [dashboard] @@ -680,6 +345,226 @@ #login_url=http://localhost/auth/login/ +[data_processing] + +# +# Options defined in tempest.config +# + +# Catalog type of the data processing service. (string value) +#catalog_type=data_processing + + +[debug] + +# +# Options defined in tempest.config +# + +# Enable diagnostic commands (boolean value) +#enable=true + + +[identity] + +# +# Options defined in tempest.config +# + +# Catalog type of the Identity service. (string value) +#catalog_type=identity + +# Set to True if using self-signed SSL certificates. (boolean +# value) +#disable_ssl_certificate_validation=false + +# Full URI of the OpenStack Identity API (Keystone), v2 +# (string value) +#uri= + +# Full URI of the OpenStack Identity API (Keystone), v3 +# (string value) +#uri_v3= + +# The identity region name to use. Also used as the other +# services' region name unless they are set explicitly. If no +# such region is found in the service catalog, the first found +# one is used. (string value) +#region=RegionOne + +# Username to use for Nova API requests. (string value) +#username=demo + +# Tenant name to use for Nova API requests. (string value) +#tenant_name=demo + +# Role required to administrate keystone. (string value) +#admin_role=admin + +# API key to use when authenticating. (string value) +#password=pass + +# Username of alternate user to use for Nova API requests. +# (string value) +#alt_username= + +# Alternate user's Tenant name to use for Nova API requests. +# (string value) +#alt_tenant_name= + +# API key to use when authenticating as alternate user. +# (string value) +#alt_password= + +# Administrative Username to use forKeystone API requests. +# (string value) +#admin_username=admin + +# Administrative Tenant name to use for Keystone API requests. +# (string value) +#admin_tenant_name=admin + +# API key to use when authenticating as admin. (string value) +#admin_password=pass + + +[image] + +# +# Options defined in tempest.config +# + +# Catalog type of the Image service. (string value) +#catalog_type=image + +# The image region name to use. If empty, the value of +# identity.region is used instead. If no such region is found +# in the service catalog, the first found one is used. (string +# value) +#region= + +# http accessible image (string value) +#http_image=http://download.cirros-cloud.net/0.3.1/cirros-0.3.1-x86_64-uec.tar.gz + + +[image-feature-enabled] + +# +# Options defined in tempest.config +# + +# Is the v2 image API enabled (boolean value) +#api_v2=true + +# Is the v1 image API enabled (boolean value) +#api_v1=true + + +[network] + +# +# Options defined in tempest.config +# + +# Catalog type of the Neutron service. (string value) +#catalog_type=network + +# The network region name to use. If empty, the value of +# identity.region is used instead. If no such region is found +# in the service catalog, the first found one is used. (string +# value) +#region= + +# The cidr block to allocate tenant networks from (string +# value) +#tenant_network_cidr=10.100.0.0/16 + +# The mask bits for tenant networks (integer value) +#tenant_network_mask_bits=28 + +# Whether tenant network connectivity should be evaluated +# directly (boolean value) +#tenant_networks_reachable=false + +# Id of the public network that provides external connectivity +# (string value) +#public_network_id= + +# Id of the public router that provides external connectivity +# (string value) +#public_router_id= + + +[network-feature-enabled] + +# +# Options defined in tempest.config +# + +# A list of enabled extensions with a special entry all which +# indicates every extension is enabled (list value) +#api_extensions=all + +# A list of enabled extensions with a special entry all which +# indicates every extension is enabled (list value) +#api_extensions=all + +# A list of enabled extensions with a special entry all which +# indicates every extension is enabled (list value) +#api_extensions=all + + +[object-storage] + +# +# Options defined in tempest.config +# + +# Catalog type of the Object-Storage service. (string value) +#catalog_type=object-store + +# The object-storage region name to use. If empty, the value +# of identity.region is used instead. If no such region is +# found in the service catalog, the first found one is used. +# (string value) +#region= + +# Number of seconds to time on waiting for a containerto +# container synchronization complete. (integer value) +#container_sync_timeout=120 + +# Number of seconds to wait while looping to check thestatus +# of a container to container synchronization (integer value) +#container_sync_interval=5 + +# Role to add to users created for swift tests to enable +# creating containers (string value) +#operator_role=Member + + +[object-storage-feature-enabled] + +# +# Options defined in tempest.config +# + +# Set to True if the Container Quota middleware is enabled +# (boolean value) +#container_quotas=true + +# Set to True if the Account Quota middleware is enabled +# (boolean value) +#accounts_quotas=true + +# Set to True if the Crossdomain middleware is enabled +# (boolean value) +#crossdomain=true + +# Set to True if the TempURL middleware is enabled (boolean +# value) +#tempurl=true + + [orchestration] # @@ -695,12 +580,6 @@ # value) #region= -# Allows test cases to create/destroy tenants and users. This -# option enables isolated test cases and better parallel -# execution, but also requires that OpenStack Identity API -# admin credentials are known. (boolean value) -#allow_tenant_isolation=false - # Time in seconds between build status checks. (integer value) #build_interval=1 @@ -725,14 +604,30 @@ #max_template_size=524288 -[debug] +[scenario] # # Options defined in tempest.config # -# Enable diagnostic commands (boolean value) -#enable=true +# Directory containing image files (string value) +#img_dir=/opt/stack/new/devstack/files/images/cirros-0.3.1-x86_64-uec + +# AMI image file name (string value) +#ami_img_file=cirros-0.3.1-x86_64-blank.img + +# ARI image file name (string value) +#ari_img_file=cirros-0.3.1-x86_64-initrd + +# AKI image file name (string value) +#aki_img_file=cirros-0.3.1-x86_64-vmlinuz + +# ssh username for the image file (string value) +#ssh_user=cirros + +# specifies how many resources to request at once. Used for +# large operations testing. (integer value) +#large_ops_number=0 [service_available] @@ -778,3 +673,109 @@ #savanna=false +[stress] + +# +# Options defined in tempest.config +# + +# Directory containing log files on the compute nodes (string +# value) +#nova_logdir= + +# Maximum number of instances to create during test. (integer +# value) +#max_instances=16 + +# Controller host. (string value) +#controller= + +# Controller host. (string value) +#target_controller= + +# ssh user. (string value) +#target_ssh_user= + +# Path to private key. (string value) +#target_private_key_path= + +# regexp for list of log files. (string value) +#target_logfiles= + +# time (in seconds) between log file error checks. (integer +# value) +#log_check_interval=60 + +# The number of threads created while stress test. (integer +# value) +#default_thread_number_per_action=4 + + +[telemetry] + +# +# Options defined in tempest.config +# + +# Catalog type of the Telemetry service. (string value) +#catalog_type=metering + + +[volume] + +# +# Options defined in tempest.config +# + +# Time in seconds between volume availability checks. (integer +# value) +#build_interval=10 + +# Timeout in seconds to wait for a volume to becomeavailable. +# (integer value) +#build_timeout=300 + +# Catalog type of the Volume Service (string value) +#catalog_type=volume + +# The volume region name to use. If empty, the value of +# identity.region is used instead. If no such region is found +# in the service catalog, the first found one is used. (string +# value) +#region= + +# Name of the backend1 (must be declared in cinder.conf) +# (string value) +#backend1_name=BACKEND_1 + +# Name of the backend2 (must be declared in cinder.conf) +# (string value) +#backend2_name=BACKEND_2 + +# Backend protocol to target when creating volume types +# (string value) +#storage_protocol=iSCSI + +# Backend vendor to target when creating volume types (string +# value) +#vendor_name=Open Source + +# Disk format to use when copying a volume to image (string +# value) +#disk_format=raw + + +[volume-feature-enabled] + +# +# Options defined in tempest.config +# + +# Runs Cinder multi-backend test (requires 2 backends) +# (boolean value) +#multi_backend=false + +# Is the v1 volume API enabled (boolean value) +#api_v1=true + + diff --git a/tempest/openstack/common/config/generator.py b/tempest/openstack/common/config/generator.py index 373f9a683e..eeb5a32839 100644 --- a/tempest/openstack/common/config/generator.py +++ b/tempest/openstack/common/config/generator.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 SINA Corporation # All Rights Reserved. # @@ -28,6 +26,7 @@ import sys import textwrap from oslo.config import cfg +import six from tempest.openstack.common import gettextutils from tempest.openstack.common import importutils @@ -78,12 +77,15 @@ def generate(srcfiles): # The options list is a list of (module, options) tuples opts_by_group = {'DEFAULT': []} - for module_name in os.getenv( - "OSLO_CONFIG_GENERATOR_EXTRA_MODULES", "").split(','): - module = _import_module(module_name) - if module: - for group, opts in _list_opts(module): - opts_by_group.setdefault(group, []).append((module_name, opts)) + extra_modules = os.getenv("TEMPEST_CONFIG_GENERATOR_EXTRA_MODULES", "") + if extra_modules: + for module_name in extra_modules.split(','): + module_name = module_name.strip() + module = _import_module(module_name) + if module: + for group, opts in _list_opts(module): + opts_by_group.setdefault(group, []).append((module_name, + opts)) for pkg_name in pkg_names: mods = mods_by_pkg.get(pkg_name) @@ -100,8 +102,8 @@ def generate(srcfiles): opts_by_group.setdefault(group, []).append((mod_str, opts)) print_group_opts('DEFAULT', opts_by_group.pop('DEFAULT', [])) - for group, opts in opts_by_group.items(): - print_group_opts(group, opts) + for group in sorted(opts_by_group.keys()): + print_group_opts(group, opts_by_group[group]) def _import_module(mod_str): @@ -111,17 +113,17 @@ def _import_module(mod_str): return sys.modules[mod_str[4:]] else: return importutils.import_module(mod_str) - except ImportError as ie: - sys.stderr.write("%s\n" % str(ie)) - return None - except Exception: + except Exception as e: + sys.stderr.write("Error importing module %s: %s\n" % (mod_str, str(e))) return None def _is_in_group(opt, group): "Check if opt is in group." for key, value in group._opts.items(): - if value['opt'] == opt: + # NOTE(llu): Temporary workaround for bug #1262148, wait until + # newly released oslo.config support '==' operator. + if not(value['opt'] != opt): return True return False @@ -233,7 +235,7 @@ def _print_opt(opt): if opt_default is None: print('#%s=' % opt_name) elif opt_type == STROPT: - assert(isinstance(opt_default, basestring)) + assert(isinstance(opt_default, six.string_types)) print('#%s=%s' % (opt_name, _sanitize_default(opt_name, opt_default))) elif opt_type == BOOLOPT: diff --git a/tempest/openstack/common/excutils.py b/tempest/openstack/common/excutils.py index c7bce72738..dc365da5ac 100644 --- a/tempest/openstack/common/excutils.py +++ b/tempest/openstack/common/excutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # Copyright 2012, Red Hat, Inc. # @@ -26,7 +24,7 @@ import traceback import six -from tempest.openstack.common.gettextutils import _ # noqa +from tempest.openstack.common.gettextutils import _ class save_and_reraise_exception(object): @@ -44,13 +42,13 @@ class save_and_reraise_exception(object): In some cases the caller may not want to re-raise the exception, and for those circumstances this context provides a reraise flag that - can be used to suppress the exception. For example: + can be used to suppress the exception. For example:: - except Exception: - with save_and_reraise_exception() as ctxt: - decide_if_need_reraise() - if not should_be_reraised: - ctxt.reraise = False + except Exception: + with save_and_reraise_exception() as ctxt: + decide_if_need_reraise() + if not should_be_reraised: + ctxt.reraise = False """ def __init__(self): self.reraise = True diff --git a/tempest/openstack/common/fileutils.py b/tempest/openstack/common/fileutils.py index 15530afab4..1845ed2a85 100644 --- a/tempest/openstack/common/fileutils.py +++ b/tempest/openstack/common/fileutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -22,7 +20,7 @@ import os import tempfile from tempest.openstack.common import excutils -from tempest.openstack.common.gettextutils import _ # noqa +from tempest.openstack.common.gettextutils import _ from tempest.openstack.common import log as logging LOG = logging.getLogger(__name__) diff --git a/tempest/openstack/common/fixture/config.py b/tempest/openstack/common/fixture/config.py index 7b044ef749..0bf90ff7a0 100644 --- a/tempest/openstack/common/fixture/config.py +++ b/tempest/openstack/common/fixture/config.py @@ -1,4 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 # # Copyright 2013 Mirantis, Inc. # Copyright 2013 OpenStack Foundation @@ -30,7 +29,7 @@ class Config(fixtures.Fixture): the specified configuration option group. All overrides are automatically cleared at the end of the current - test by the reset() method, which is registred by addCleanup(). + test by the reset() method, which is registered by addCleanup(). """ def __init__(self, conf=cfg.CONF): diff --git a/tempest/openstack/common/fixture/lockutils.py b/tempest/openstack/common/fixture/lockutils.py index 21b4a48edc..593668734c 100644 --- a/tempest/openstack/common/fixture/lockutils.py +++ b/tempest/openstack/common/fixture/lockutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -17,7 +15,7 @@ import fixtures -from tempest.openstack.common.lockutils import lock +from tempest.openstack.common import lockutils class LockFixture(fixtures.Fixture): @@ -45,7 +43,7 @@ class LockFixture(fixtures.Fixture): test method exits. (either by completing or raising an exception) """ def __init__(self, name, lock_file_prefix=None): - self.mgr = lock(name, lock_file_prefix, True) + self.mgr = lockutils.lock(name, lock_file_prefix, True) def setUp(self): super(LockFixture, self).setUp() diff --git a/tempest/openstack/common/fixture/mockpatch.py b/tempest/openstack/common/fixture/mockpatch.py index cd0d6ca6b5..858e77cd06 100644 --- a/tempest/openstack/common/fixture/mockpatch.py +++ b/tempest/openstack/common/fixture/mockpatch.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # Copyright 2013 Hewlett-Packard Development Company, L.P. diff --git a/tempest/openstack/common/fixture/moxstubout.py b/tempest/openstack/common/fixture/moxstubout.py index a0e74fd11e..e8c031f08a 100644 --- a/tempest/openstack/common/fixture/moxstubout.py +++ b/tempest/openstack/common/fixture/moxstubout.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # Copyright 2013 Hewlett-Packard Development Company, L.P. diff --git a/tempest/openstack/common/gettextutils.py b/tempest/openstack/common/gettextutils.py index 2939ed9fd0..825c2e01df 100644 --- a/tempest/openstack/common/gettextutils.py +++ b/tempest/openstack/common/gettextutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 Red Hat, Inc. # Copyright 2013 IBM Corp. # All Rights Reserved. @@ -317,7 +315,7 @@ def get_available_languages(domain): # NOTE(luisg): Babel <1.0 used a function called list(), which was # renamed to locale_identifiers() in >=1.0, the requirements master list # requires >=0.9.6, uncapped, so defensively work with both. We can remove - # this check when the master list updates to >=1.0, and all projects udpate + # this check when the master list updates to >=1.0, and update all projects list_identifiers = (getattr(localedata, 'list', None) or getattr(localedata, 'locale_identifiers')) locale_identifiers = list_identifiers() diff --git a/tempest/openstack/common/importutils.py b/tempest/openstack/common/importutils.py index 7a303f93f2..4fd9ae2bc2 100644 --- a/tempest/openstack/common/importutils.py +++ b/tempest/openstack/common/importutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # diff --git a/tempest/openstack/common/jsonutils.py b/tempest/openstack/common/jsonutils.py index b589545313..53c0ad4048 100644 --- a/tempest/openstack/common/jsonutils.py +++ b/tempest/openstack/common/jsonutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # Copyright 2011 Justin Santa Barbara @@ -41,8 +39,12 @@ import json try: import xmlrpclib except ImportError: - # NOTE(jd): xmlrpclib is not shipped with Python 3 - xmlrpclib = None + # NOTE(jaypipes): xmlrpclib was renamed to xmlrpc.client in Python3 + # however the function and object call signatures + # remained the same. This whole try/except block should + # be removed and replaced with a call to six.moves once + # six 1.4.2 is released. See http://bit.ly/1bqrVzu + import xmlrpc.client as xmlrpclib import six @@ -124,14 +126,14 @@ def to_primitive(value, convert_instances=False, convert_datetime=True, level=level, max_depth=max_depth) if isinstance(value, dict): - return dict((k, recursive(v)) for k, v in value.iteritems()) + return dict((k, recursive(v)) for k, v in six.iteritems(value)) elif isinstance(value, (list, tuple)): return [recursive(lv) for lv in value] # It's not clear why xmlrpclib created their own DateTime type, but # for our purposes, make it a datetime type which is explicitly # handled - if xmlrpclib and isinstance(value, xmlrpclib.DateTime): + if isinstance(value, xmlrpclib.DateTime): value = datetime.datetime(*tuple(value.timetuple())[:6]) if convert_datetime and isinstance(value, datetime.datetime): diff --git a/tempest/openstack/common/local.py b/tempest/openstack/common/local.py index e82f17d0f3..0819d5b97c 100644 --- a/tempest/openstack/common/local.py +++ b/tempest/openstack/common/local.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # diff --git a/tempest/openstack/common/lockutils.py b/tempest/openstack/common/lockutils.py index 65f3548477..53cada1b0c 100644 --- a/tempest/openstack/common/lockutils.py +++ b/tempest/openstack/common/lockutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -31,7 +29,7 @@ import weakref from oslo.config import cfg from tempest.openstack.common import fileutils -from tempest.openstack.common.gettextutils import _ # noqa +from tempest.openstack.common.gettextutils import _ from tempest.openstack.common import local from tempest.openstack.common import log as logging @@ -148,16 +146,16 @@ def lock(name, lock_file_prefix=None, external=False, lock_path=None): True, in which case, it'll yield an InterProcessLock instance. :param lock_file_prefix: The lock_file_prefix argument is used to provide - lock files on disk with a meaningful prefix. + lock files on disk with a meaningful prefix. :param external: The external keyword argument denotes whether this lock - should work across multiple processes. This means that if two different - workers both run a a method decorated with @synchronized('mylock', - external=True), only one of them will execute at a time. + should work across multiple processes. This means that if two different + workers both run a a method decorated with @synchronized('mylock', + external=True), only one of them will execute at a time. :param lock_path: The lock_path keyword argument is used to specify a - special location for external lock files to live. If nothing is set, then - CONF.lock_path is used as a default. + special location for external lock files to live. If nothing is set, then + CONF.lock_path is used as a default. """ with _semaphores_lock: try: diff --git a/tempest/openstack/common/log.py b/tempest/openstack/common/log.py index abb44efb66..7bebfdbacc 100644 --- a/tempest/openstack/common/log.py +++ b/tempest/openstack/common/log.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. @@ -35,6 +33,7 @@ import logging import logging.config import logging.handlers import os +import re import sys import traceback @@ -42,7 +41,7 @@ from oslo.config import cfg import six from six import moves -from tempest.openstack.common.gettextutils import _ # noqa +from tempest.openstack.common.gettextutils import _ from tempest.openstack.common import importutils from tempest.openstack.common import jsonutils from tempest.openstack.common import local @@ -50,6 +49,24 @@ from tempest.openstack.common import local _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" +_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password'] + +# NOTE(ldbragst): Let's build a list of regex objects using the list of +# _SANITIZE_KEYS we already have. This way, we only have to add the new key +# to the list of _SANITIZE_KEYS and we can generate regular expressions +# for XML and JSON automatically. +_SANITIZE_PATTERNS = [] +_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])', + r'(<%(key)s>).*?()', + r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])', + r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])'] + +for key in _SANITIZE_KEYS: + for pattern in _FORMAT_PATTERNS: + reg_ex = re.compile(pattern % {'key': key}, re.DOTALL) + _SANITIZE_PATTERNS.append(reg_ex) + + common_cli_opts = [ cfg.BoolOpt('debug', short='d', @@ -64,11 +81,13 @@ common_cli_opts = [ ] logging_cli_opts = [ - cfg.StrOpt('log-config', + cfg.StrOpt('log-config-append', metavar='PATH', - help='If this option is specified, the logging configuration ' - 'file specified is used and overrides any other logging ' - 'options specified. Please see the Python logging module ' + deprecated_name='log-config', + help='The name of logging configuration file. It does not ' + 'disable existing loggers, but just appends specified ' + 'logging configuration to any other existing logging ' + 'options. Please see the Python logging module ' 'documentation for details on logging configuration ' 'files.'), cfg.StrOpt('log-format', @@ -111,7 +130,7 @@ generic_log_opts = [ log_opts = [ cfg.StrOpt('logging_context_format_string', default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' - '%(name)s [%(request_id)s %(user)s %(tenant)s] ' + '%(name)s [%(request_id)s %(user_identity)s] ' '%(instance)s%(message)s', help='format string to use for log messages with context'), cfg.StrOpt('logging_default_format_string', @@ -127,12 +146,13 @@ log_opts = [ help='prefix each line of exception output with this format'), cfg.ListOpt('default_log_levels', default=[ + 'amqp=WARN', 'amqplib=WARN', - 'sqlalchemy=WARN', 'boto=WARN', + 'qpid=WARN', + 'sqlalchemy=WARN', 'suds=INFO', - 'keystone=INFO', - 'paramiko=INFO' + 'iso8601=WARN', ], help='list of logger=LEVEL pairs'), cfg.BoolOpt('publish_errors', @@ -211,6 +231,40 @@ def _get_log_file_path(binary=None): return None +def mask_password(message, secret="***"): + """Replace password with 'secret' in message. + + :param message: The string which includes security information. + :param secret: value with which to replace passwords. + :returns: The unicode value of message with the password fields masked. + + For example: + + >>> mask_password("'adminPass' : 'aaaaa'") + "'adminPass' : '***'" + >>> mask_password("'admin_pass' : 'aaaaa'") + "'admin_pass' : '***'" + >>> mask_password('"password" : "aaaaa"') + '"password" : "***"' + >>> mask_password("'original_password' : 'aaaaa'") + "'original_password' : '***'" + >>> mask_password("u'original_password' : u'aaaaa'") + "u'original_password' : u'***'" + """ + message = six.text_type(message) + + # NOTE(ldbragst): Check to see if anything in message contains any key + # specified in _SANITIZE_KEYS, if not then just return the message since + # we don't have to mask any passwords. + if not any(key in message for key in _SANITIZE_KEYS): + return message + + secret = r'\g<1>' + secret + r'\g<2>' + for pattern in _SANITIZE_PATTERNS: + message = re.sub(pattern, secret, message) + return message + + class BaseLoggerAdapter(logging.LoggerAdapter): def audit(self, msg, *args, **kwargs): @@ -278,10 +332,12 @@ class ContextAdapter(BaseLoggerAdapter): elif instance_uuid: instance_extra = (CONF.instance_uuid_format % {'uuid': instance_uuid}) - extra.update({'instance': instance_extra}) + extra['instance'] = instance_extra - extra.update({"project": self.project}) - extra.update({"version": self.version}) + extra.setdefault('user_identity', kwargs.pop('user_identity', None)) + + extra['project'] = self.project + extra['version'] = self.version extra['extra'] = extra.copy() return msg, kwargs @@ -295,7 +351,7 @@ class JSONFormatter(logging.Formatter): def formatException(self, ei, strip_newlines=True): lines = traceback.format_exception(*ei) if strip_newlines: - lines = [itertools.ifilter( + lines = [moves.filter( lambda x: x, line.rstrip().splitlines()) for line in lines] lines = list(itertools.chain(*lines)) @@ -333,10 +389,10 @@ class JSONFormatter(logging.Formatter): def _create_logging_excepthook(product_name): - def logging_excepthook(type, value, tb): + def logging_excepthook(exc_type, value, tb): extra = {} if CONF.verbose: - extra['exc_info'] = (type, value, tb) + extra['exc_info'] = (exc_type, value, tb) getLogger(product_name).critical(str(value), **extra) return logging_excepthook @@ -354,17 +410,18 @@ class LogConfigError(Exception): err_msg=self.err_msg) -def _load_log_config(log_config): +def _load_log_config(log_config_append): try: - logging.config.fileConfig(log_config) + logging.config.fileConfig(log_config_append, + disable_existing_loggers=False) except moves.configparser.Error as exc: - raise LogConfigError(log_config, str(exc)) + raise LogConfigError(log_config_append, str(exc)) def setup(product_name): """Setup logging.""" - if CONF.log_config: - _load_log_config(CONF.log_config) + if CONF.log_config_append: + _load_log_config(CONF.log_config_append) else: _setup_logging_from_conf() sys.excepthook = _create_logging_excepthook(product_name) @@ -420,7 +477,7 @@ def _setup_logging_from_conf(): streamlog = ColorHandler() log_root.addHandler(streamlog) - elif not CONF.log_file: + elif not logpath: # pass sys.stdout as a positional argument # python2.6 calls the argument strm, in 2.7 it's stream streamlog = logging.StreamHandler(sys.stdout) diff --git a/tempest/openstack/common/timeutils.py b/tempest/openstack/common/timeutils.py index 98d877d59a..d5ed81d3e3 100644 --- a/tempest/openstack/common/timeutils.py +++ b/tempest/openstack/common/timeutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -50,9 +48,9 @@ def parse_isotime(timestr): try: return iso8601.parse_date(timestr) except iso8601.ParseError as e: - raise ValueError(unicode(e)) + raise ValueError(six.text_type(e)) except TypeError as e: - raise ValueError(unicode(e)) + raise ValueError(six.text_type(e)) def strtime(at=None, fmt=PERFECT_TIME_FORMAT): @@ -79,6 +77,9 @@ def is_older_than(before, seconds): """Return True if before is older than seconds.""" if isinstance(before, six.string_types): before = parse_strtime(before).replace(tzinfo=None) + else: + before = before.replace(tzinfo=None) + return utcnow() - before > datetime.timedelta(seconds=seconds) @@ -86,6 +87,9 @@ def is_newer_than(after, seconds): """Return True if after is newer than seconds.""" if isinstance(after, six.string_types): after = parse_strtime(after).replace(tzinfo=None) + else: + after = after.replace(tzinfo=None) + return after - utcnow() > datetime.timedelta(seconds=seconds) @@ -178,6 +182,15 @@ def delta_seconds(before, after): datetime objects (as a float, to microsecond resolution). """ delta = after - before + return total_seconds(delta) + + +def total_seconds(delta): + """Return the total seconds of datetime.timedelta object. + + Compute total seconds of datetime.timedelta, datetime.timedelta + doesn't have method total_seconds in Python2.6, calculate it manually. + """ try: return delta.total_seconds() except AttributeError: @@ -188,8 +201,8 @@ def delta_seconds(before, after): def is_soon(dt, window): """Determines if time is going to happen in the next window seconds. - :params dt: the time - :params window: minimum seconds to remain to consider the time not soon + :param dt: the time + :param window: minimum seconds to remain to consider the time not soon :return: True if expiration is within the given duration """ diff --git a/tools/config/generate_sample.sh b/tools/config/generate_sample.sh index b86e0c2e68..607fecb6ae 100755 --- a/tools/config/generate_sample.sh +++ b/tools/config/generate_sample.sh @@ -91,3 +91,9 @@ DEFAULT_MODULEPATH=tempest.openstack.common.config.generator MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH} OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample python -m $MODULEPATH $FILES > $OUTPUTFILE + +# Hook to allow projects to append custom config file snippets +CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null) +for CONCAT_FILE in $CONCAT_FILES; do + cat $CONCAT_FILE >> $OUTPUTFILE +done diff --git a/tools/install_venv_common.py b/tools/install_venv_common.py index 1bab88a3fd..46822e3293 100644 --- a/tools/install_venv_common.py +++ b/tools/install_venv_common.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2013 OpenStack Foundation # Copyright 2013 IBM Corp. #