openstack/tempest
Code Issues Proposed changes

sync oslo to current

Browse Source 
this is largely to deal with config generator issues, however
we're apparently quite far behind on other items, so lets just
sync it all.

Change-Id: I2f36e41b5d7cce8dd0b64636e45da3f7481b0e39
This commit is contained in:
Sean Dague
2014-01-03 08:51:54 -05:00
parent c4cf2973f1
commit fc691e33f3
17 changed files with 596 additions and 536 deletions
Download Patch File Download Diff File Expand all files Collapse all files

891
etc/tempest.conf.sample
View File

@@ -28,7 +28,7 @@
# format string to use for log messages with context (string # format string to use for log messages with context (string
# value) # value)
#logging_context_format_string=%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user)s %(tenant)s] %(instance)s%(message)s #logging_context_format_string=%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s
# format string to use for log messages without context # format string to use for log messages without context
# (string value) # (string value)
@@ -43,7 +43,7 @@
#logging_exception_prefix=%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s #logging_exception_prefix=%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s
# list of logger=LEVEL pairs (list value) # list of logger=LEVEL pairs (list value)
#default_log_levels=amqplib=WARN,sqlalchemy=WARN,boto=WARN,suds=INFO,keystone=INFO,paramiko=INFO #default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,iso8601=WARN
# publish error events (boolean value) # publish error events (boolean value)
#publish_errors=false #publish_errors=false
@@ -59,12 +59,13 @@
# it like this (string value) # it like this (string value)
#instance_uuid_format="[instance: %(uuid)s] " #instance_uuid_format="[instance: %(uuid)s] "
# If this option is specified, the logging configuration file # The name of logging configuration file. It does not disable
# specified is used and overrides any other logging options # existing loggers, but just appends specified logging
# specified. Please see the Python logging module # configuration to any other existing logging options. Please
# documentation for details on logging configuration files. # see the Python logging module documentation for details on
# (string value) # logging configuration files. (string value)
#log_config=<None> # Deprecated group/name - [DEFAULT]/log_config
#log_config_append=<None>
# DEPRECATED. A logging.Formatter log message format string # DEPRECATED. A logging.Formatter log message format string
# which may use any of the available logging.LogRecord # which may use any of the available logging.LogRecord
@@ -94,377 +95,67 @@
#syslog_log_facility=LOG_USER #syslog_log_facility=LOG_USER
[image] [boto]
# #
# Options defined in tempest.config # Options defined in tempest.config
# #
# Catalog type of the Image service. (string value) # EC2 URL (string value)
#catalog_type=image #ec2_url=http://localhost:8773/services/Cloud
# The image region name to use. If empty, the value of # S3 URL (string value)
# identity.region is used instead. If no such region is found #s3_url=http://localhost:8080
# in the service catalog, the first found one is used. (string
# AWS Secret Key (string value)
#aws_secret=<None>
# AWS Access Key (string value)
#aws_access=<None>
# S3 Materials Path (string value)
#s3_materials_path=/opt/stack/devstack/files/images/s3-materials/cirros-0.3.0
# ARI Ramdisk Image manifest (string value)
#ari_manifest=cirros-0.3.0-x86_64-initrd.manifest.xml
# AMI Machine Image manifest (string value)
#ami_manifest=cirros-0.3.0-x86_64-blank.img.manifest.xml
# AKI Kernel Image manifest (string value)
#aki_manifest=cirros-0.3.0-x86_64-vmlinuz.manifest.xml
# Instance type (string value)
#instance_type=m1.tiny
# boto Http socket timeout (integer value)
#http_socket_timeout=3
# boto num_retries on error (integer value)
#num_retries=1
# Status Change Timeout (integer value)
#build_timeout=60
# Status Change Test Interval (integer value)
#build_interval=1
[cli]
#
# Options defined in tempest.cli
#
# enable cli tests (boolean value)
#enabled=true
# directory where python client binaries are located (string
# value) # value)
#region= #cli_dir=/usr/local/bin
# http accessible image (string value) # Number of seconds to wait on a CLI timeout (integer value)
#http_image=http://download.cirros-cloud.net/0.3.1/cirros-0.3.1-x86_64-uec.tar.gz #timeout=15
[object-storage]
#
# Options defined in tempest.config
#
# Catalog type of the Object-Storage service. (string value)
#catalog_type=object-store
# The object-storage region name to use. If empty, the value
# of identity.region is used instead. If no such region is
# found in the service catalog, the first found one is used.
# (string value)
#region=
# Number of seconds to time on waiting for a containerto
# container synchronization complete. (integer value)
#container_sync_timeout=120
# Number of seconds to wait while looping to check thestatus
# of a container to container synchronization (integer value)
#container_sync_interval=5
# Role to add to users created for swift tests to enable
# creating containers (string value)
#operator_role=Member
[volume-feature-enabled]
#
# Options defined in tempest.config
#
# Runs Cinder multi-backend test (requires 2 backends)
# (boolean value)
#multi_backend=false
# A list of enabled extensions with a special entry all which
# indicates every extension is enabled (list value)
#api_extensions=all
# Is the v1 volume API enabled (boolean value)
#api_v1=true
[telemetry]
#
# Options defined in tempest.config
#
# Catalog type of the Telemetry service. (string value)
#catalog_type=metering
[data_processing]
#
# Options defined in tempest.config
#
# Catalog type of the data processing service. (string value)
#catalog_type=data_processing
[object-storage-feature-enabled]
#
# Options defined in tempest.config
#
# Set to True if the Container Quota middleware is enabled
# (boolean value)
#container_quotas=true
# Set to True if the Account Quota middleware is enabled
# (boolean value)
#accounts_quotas=true
# Set to True if the Crossdomain middleware is enabled
# (boolean value)
#crossdomain=true
# Set to True if the TempURL middleware is enabled (boolean
# value)
#tempurl=true
[network-feature-enabled]
#
# Options defined in tempest.config
#
# A list of enabled extensions with a special entry all which
# indicates every extension is enabled (list value)
#api_extensions=all
[image-feature-enabled]
#
# Options defined in tempest.config
#
# Is the v2 image API enabled (boolean value)
#api_v2=true
# Is the v1 image API enabled (boolean value)
#api_v1=true
[compute-admin]
#
# Options defined in tempest.config
#
# Administrative Username to use for Nova API requests.
# (string value)
#username=admin
# Administrative Tenant name to use for Nova API requests.
# (string value)
#tenant_name=admin
# API key to use when authenticating as admin. (string value)
#password=pass
[volume]
#
# Options defined in tempest.config
#
# Time in seconds between volume availability checks. (integer
# value)
#build_interval=10
# Timeout in seconds to wait for a volume to becomeavailable.
# (integer value)
#build_timeout=300
# Catalog type of the Volume Service (string value)
#catalog_type=volume
# The volume region name to use. If empty, the value of
# identity.region is used instead. If no such region is found
# in the service catalog, the first found one is used. (string
# value)
#region=
# Name of the backend1 (must be declared in cinder.conf)
# (string value)
#backend1_name=BACKEND_1
# Name of the backend2 (must be declared in cinder.conf)
# (string value)
#backend2_name=BACKEND_2
# Backend protocol to target when creating volume types
# (string value)
#storage_protocol=iSCSI
# Backend vendor to target when creating volume types (string
# value)
#vendor_name=Open Source
# Disk format to use when copying a volume to image (string
# value)
#disk_format=raw
[compute-feature-enabled]
#
# Options defined in tempest.config
#
# If false, skip all nova v3 tests. (boolean value)
#api_v3=true
# If false, skip disk config tests (boolean value)
#disk_config=true
# A list of enabled extensions with a special entry all which
# indicates every extension is enabled (list value)
#api_extensions=all
# A list of enabled v3 extensions with a special entry all
# which indicates every extension is enabled (list value)
#api_v3_extensions=all
# Does the test environment support changing the admin
# password? (boolean value)
#change_password=false
# Does the test environment support snapshots? (boolean value)
#create_image=false
# Does the test environment support resizing? (boolean value)
#resize=false
# Does the test environment support live migration available?
# (boolean value)
#live_migration=false
# Does the test environment use block devices for live
# migration (boolean value)
#block_migration_for_live_migration=false
# Does the test environment block migration support cinder
# iSCSI volumes (boolean value)
#block_migrate_cinder_iscsi=false
[identity]
#
# Options defined in tempest.config
#
# Catalog type of the Identity service. (string value)
#catalog_type=identity
# Set to True if using self-signed SSL certificates. (boolean
# value)
#disable_ssl_certificate_validation=false
# Full URI of the OpenStack Identity API (Keystone), v2
# (string value)
#uri=<None>
# Full URI of the OpenStack Identity API (Keystone), v3
# (string value)
#uri_v3=<None>
# The identity region name to use. Also used as the other
# services' region name unless they are set explicitly. If no
# such region is found in the service catalog, the first found
# one is used. (string value)
#region=RegionOne
# Username to use for Nova API requests. (string value)
#username=demo
# Tenant name to use for Nova API requests. (string value)
#tenant_name=demo
# Role required to administrate keystone. (string value)
#admin_role=admin
# API key to use when authenticating. (string value)
#password=pass
# Username of alternate user to use for Nova API requests.
# (string value)
#alt_username=<None>
# Alternate user's Tenant name to use for Nova API requests.
# (string value)
#alt_tenant_name=<None>
# API key to use when authenticating as alternate user.
# (string value)
#alt_password=<None>
# Administrative Username to use forKeystone API requests.
# (string value)
#admin_username=admin
# Administrative Tenant name to use for Keystone API requests.
# (string value)
#admin_tenant_name=admin
# API key to use when authenticating as admin. (string value)
#admin_password=pass
[network]
#
# Options defined in tempest.config
#
# Catalog type of the Neutron service. (string value)
#catalog_type=network
# The network region name to use. If empty, the value of
# identity.region is used instead. If no such region is found
# in the service catalog, the first found one is used. (string
# value)
#region=
# The cidr block to allocate tenant networks from (string
# value)
#tenant_network_cidr=10.100.0.0/16
# The mask bits for tenant networks (integer value)
#tenant_network_mask_bits=28
# Whether tenant network connectivity should be evaluated
# directly (boolean value)
#tenant_networks_reachable=false
# Id of the public network that provides external connectivity
# (string value)
#public_network_id=
# Id of the public router that provides external connectivity
# (string value)
#public_router_id=
[stress]
#
# Options defined in tempest.config
#
# Directory containing log files on the compute nodes (string
# value)
#nova_logdir=<None>
# Maximum number of instances to create during test. (integer
# value)
#max_instances=16
# Controller host. (string value)
#controller=<None>
# Controller host. (string value)
#target_controller=<None>
# ssh user. (string value)
#target_ssh_user=<None>
# Path to private key. (string value)
#target_private_key_path=<None>
# regexp for list of log files. (string value)
#target_logfiles=<None>
# time (in seconds) between log file error checks. (integer
# value)
#log_check_interval=60
# The number of threads created while stress test. (integer
# value)
#default_thread_number_per_action=4
[compute] [compute]
@@ -577,94 +268,68 @@
# (integer value) # (integer value)
#shelved_offload_time=0 #shelved_offload_time=0
# Allows test cases to create/destroy tenants and users. This
# option enables isolated test cases and better parallel
# execution, but also requires that OpenStack Identity API
# admin credentials are known. (boolean value)
#allow_tenant_isolation=false
[scenario]
[compute-admin]
# #
# Options defined in tempest.config # Options defined in tempest.config
# #
# Directory containing image files (string value) # Administrative Username to use for Nova API requests.
#img_dir=/opt/stack/new/devstack/files/images/cirros-0.3.1-x86_64-uec # (string value)
#username=admin
# AMI image file name (string value) # Administrative Tenant name to use for Nova API requests.
#ami_img_file=cirros-0.3.1-x86_64-blank.img # (string value)
#tenant_name=admin
# ARI image file name (string value) # API key to use when authenticating as admin. (string value)
#ari_img_file=cirros-0.3.1-x86_64-initrd #password=pass
# AKI image file name (string value)
#aki_img_file=cirros-0.3.1-x86_64-vmlinuz
# ssh username for the image file (string value)
#ssh_user=cirros
# specifies how many resources to request at once. Used for
# large operations testing. (integer value)
#large_ops_number=0
[boto] [compute-feature-enabled]
# #
# Options defined in tempest.config # Options defined in tempest.config
# #
# EC2 URL (string value) # If false, skip all nova v3 tests. (boolean value)
#ec2_url=http://localhost:8773/services/Cloud #api_v3=true
# S3 URL (string value) # If false, skip disk config tests (boolean value)
#s3_url=http://localhost:8080 #disk_config=true
# AWS Secret Key (string value) # A list of enabled v3 extensions with a special entry all
#aws_secret=<None> # which indicates every extension is enabled (list value)
#api_v3_extensions=all
# AWS Access Key (string value) # Does the test environment support changing the admin
#aws_access=<None> # password? (boolean value)
#change_password=false
# S3 Materials Path (string value) # Does the test environment support snapshots? (boolean value)
#s3_materials_path=/opt/stack/devstack/files/images/s3-materials/cirros-0.3.0 #create_image=false
# ARI Ramdisk Image manifest (string value) # Does the test environment support resizing? (boolean value)
#ari_manifest=cirros-0.3.0-x86_64-initrd.manifest.xml #resize=false
# AMI Machine Image manifest (string value) # Does the test environment support live migration available?
#ami_manifest=cirros-0.3.0-x86_64-blank.img.manifest.xml # (boolean value)
#live_migration=false
# AKI Kernel Image manifest (string value) # Does the test environment use block devices for live
#aki_manifest=cirros-0.3.0-x86_64-vmlinuz.manifest.xml # migration (boolean value)
#block_migration_for_live_migration=false
# Instance type (string value) # Does the test environment block migration support cinder
#instance_type=m1.tiny # iSCSI volumes (boolean value)
#block_migrate_cinder_iscsi=false
# boto Http socket timeout (integer value)
#http_socket_timeout=3
# boto num_retries on error (integer value)
#num_retries=1
# Status Change Timeout (integer value)
#build_timeout=60
# Status Change Test Interval (integer value)
#build_interval=1
[cli]
#
# Options defined in tempest.cli
#
# enable cli tests (boolean value)
#enabled=true
# directory where python client binaries are located (string
# value)
#cli_dir=/usr/local/bin
# Number of seconds to wait on a CLI timeout (integer value)
#timeout=15
[dashboard] [dashboard]
@@ -680,6 +345,226 @@
#login_url=http://localhost/auth/login/ #login_url=http://localhost/auth/login/
[data_processing]
#
# Options defined in tempest.config
#
# Catalog type of the data processing service. (string value)
#catalog_type=data_processing
[debug]
#
# Options defined in tempest.config
#
# Enable diagnostic commands (boolean value)
#enable=true
[identity]
#
# Options defined in tempest.config
#
# Catalog type of the Identity service. (string value)
#catalog_type=identity
# Set to True if using self-signed SSL certificates. (boolean
# value)
#disable_ssl_certificate_validation=false
# Full URI of the OpenStack Identity API (Keystone), v2
# (string value)
#uri=<None>
# Full URI of the OpenStack Identity API (Keystone), v3
# (string value)
#uri_v3=<None>
# The identity region name to use. Also used as the other
# services' region name unless they are set explicitly. If no
# such region is found in the service catalog, the first found
# one is used. (string value)
#region=RegionOne
# Username to use for Nova API requests. (string value)
#username=demo
# Tenant name to use for Nova API requests. (string value)
#tenant_name=demo
# Role required to administrate keystone. (string value)
#admin_role=admin
# API key to use when authenticating. (string value)
#password=pass
# Username of alternate user to use for Nova API requests.
# (string value)
#alt_username=<None>
# Alternate user's Tenant name to use for Nova API requests.
# (string value)
#alt_tenant_name=<None>
# API key to use when authenticating as alternate user.
# (string value)
#alt_password=<None>
# Administrative Username to use forKeystone API requests.
# (string value)
#admin_username=admin
# Administrative Tenant name to use for Keystone API requests.
# (string value)
#admin_tenant_name=admin
# API key to use when authenticating as admin. (string value)
#admin_password=pass
[image]
#
# Options defined in tempest.config
#
# Catalog type of the Image service. (string value)
#catalog_type=image
# The image region name to use. If empty, the value of
# identity.region is used instead. If no such region is found
# in the service catalog, the first found one is used. (string
# value)
#region=
# http accessible image (string value)
#http_image=http://download.cirros-cloud.net/0.3.1/cirros-0.3.1-x86_64-uec.tar.gz
[image-feature-enabled]
#
# Options defined in tempest.config
#
# Is the v2 image API enabled (boolean value)
#api_v2=true
# Is the v1 image API enabled (boolean value)
#api_v1=true
[network]
#
# Options defined in tempest.config
#
# Catalog type of the Neutron service. (string value)
#catalog_type=network
# The network region name to use. If empty, the value of
# identity.region is used instead. If no such region is found
# in the service catalog, the first found one is used. (string
# value)
#region=
# The cidr block to allocate tenant networks from (string
# value)
#tenant_network_cidr=10.100.0.0/16
# The mask bits for tenant networks (integer value)
#tenant_network_mask_bits=28
# Whether tenant network connectivity should be evaluated
# directly (boolean value)
#tenant_networks_reachable=false
# Id of the public network that provides external connectivity
# (string value)
#public_network_id=
# Id of the public router that provides external connectivity
# (string value)
#public_router_id=
[network-feature-enabled]
#
# Options defined in tempest.config
#
# A list of enabled extensions with a special entry all which
# indicates every extension is enabled (list value)
#api_extensions=all
# A list of enabled extensions with a special entry all which
# indicates every extension is enabled (list value)
#api_extensions=all
# A list of enabled extensions with a special entry all which
# indicates every extension is enabled (list value)
#api_extensions=all
[object-storage]
#
# Options defined in tempest.config
#
# Catalog type of the Object-Storage service. (string value)
#catalog_type=object-store
# The object-storage region name to use. If empty, the value
# of identity.region is used instead. If no such region is
# found in the service catalog, the first found one is used.
# (string value)
#region=
# Number of seconds to time on waiting for a containerto
# container synchronization complete. (integer value)
#container_sync_timeout=120
# Number of seconds to wait while looping to check thestatus
# of a container to container synchronization (integer value)
#container_sync_interval=5
# Role to add to users created for swift tests to enable
# creating containers (string value)
#operator_role=Member
[object-storage-feature-enabled]
#
# Options defined in tempest.config
#
# Set to True if the Container Quota middleware is enabled
# (boolean value)
#container_quotas=true
# Set to True if the Account Quota middleware is enabled
# (boolean value)
#accounts_quotas=true
# Set to True if the Crossdomain middleware is enabled
# (boolean value)
#crossdomain=true
# Set to True if the TempURL middleware is enabled (boolean
# value)
#tempurl=true
[orchestration] [orchestration]
# #
@@ -695,12 +580,6 @@
# value) # value)
#region= #region=
# Allows test cases to create/destroy tenants and users. This
# option enables isolated test cases and better parallel
# execution, but also requires that OpenStack Identity API
# admin credentials are known. (boolean value)
#allow_tenant_isolation=false
# Time in seconds between build status checks. (integer value) # Time in seconds between build status checks. (integer value)
#build_interval=1 #build_interval=1
@@ -725,14 +604,30 @@
#max_template_size=524288 #max_template_size=524288
[debug] [scenario]
# #
# Options defined in tempest.config # Options defined in tempest.config
# #
# Enable diagnostic commands (boolean value) # Directory containing image files (string value)
#enable=true #img_dir=/opt/stack/new/devstack/files/images/cirros-0.3.1-x86_64-uec
# AMI image file name (string value)
#ami_img_file=cirros-0.3.1-x86_64-blank.img
# ARI image file name (string value)
#ari_img_file=cirros-0.3.1-x86_64-initrd
# AKI image file name (string value)
#aki_img_file=cirros-0.3.1-x86_64-vmlinuz
# ssh username for the image file (string value)
#ssh_user=cirros
# specifies how many resources to request at once. Used for
# large operations testing. (integer value)
#large_ops_number=0
[service_available] [service_available]
@@ -778,3 +673,109 @@
#savanna=false #savanna=false
[stress]
#
# Options defined in tempest.config
#
# Directory containing log files on the compute nodes (string
# value)
#nova_logdir=<None>
# Maximum number of instances to create during test. (integer
# value)
#max_instances=16
# Controller host. (string value)
#controller=<None>
# Controller host. (string value)
#target_controller=<None>
# ssh user. (string value)
#target_ssh_user=<None>
# Path to private key. (string value)
#target_private_key_path=<None>
# regexp for list of log files. (string value)
#target_logfiles=<None>
# time (in seconds) between log file error checks. (integer
# value)
#log_check_interval=60
# The number of threads created while stress test. (integer
# value)
#default_thread_number_per_action=4
[telemetry]
#
# Options defined in tempest.config
#
# Catalog type of the Telemetry service. (string value)
#catalog_type=metering
[volume]
#
# Options defined in tempest.config
#
# Time in seconds between volume availability checks. (integer
# value)
#build_interval=10
# Timeout in seconds to wait for a volume to becomeavailable.
# (integer value)
#build_timeout=300
# Catalog type of the Volume Service (string value)
#catalog_type=volume
# The volume region name to use. If empty, the value of
# identity.region is used instead. If no such region is found
# in the service catalog, the first found one is used. (string
# value)
#region=
# Name of the backend1 (must be declared in cinder.conf)
# (string value)
#backend1_name=BACKEND_1
# Name of the backend2 (must be declared in cinder.conf)
# (string value)
#backend2_name=BACKEND_2
# Backend protocol to target when creating volume types
# (string value)
#storage_protocol=iSCSI
# Backend vendor to target when creating volume types (string
# value)
#vendor_name=Open Source
# Disk format to use when copying a volume to image (string
# value)
#disk_format=raw
[volume-feature-enabled]
#
# Options defined in tempest.config
#
# Runs Cinder multi-backend test (requires 2 backends)
# (boolean value)
#multi_backend=false
# Is the v1 volume API enabled (boolean value)
#api_v1=true

28
tempest/openstack/common/config/generator.py
View File

@@ -1,5 +1,3 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2012 SINA Corporation # Copyright 2012 SINA Corporation
# All Rights Reserved. # All Rights Reserved.
# #
@@ -28,6 +26,7 @@ import sys
import textwrap import textwrap
from oslo.config import cfg from oslo.config import cfg
import six
from tempest.openstack.common import gettextutils from tempest.openstack.common import gettextutils
from tempest.openstack.common import importutils from tempest.openstack.common import importutils
@@ -78,12 +77,15 @@ def generate(srcfiles):
# The options list is a list of (module, options) tuples # The options list is a list of (module, options) tuples
opts_by_group = {'DEFAULT': []} opts_by_group = {'DEFAULT': []}
for module_name in os.getenv( extra_modules = os.getenv("TEMPEST_CONFIG_GENERATOR_EXTRA_MODULES", "")
"OSLO_CONFIG_GENERATOR_EXTRA_MODULES", "").split(','): if extra_modules:
for module_name in extra_modules.split(','):
module_name = module_name.strip()
module = _import_module(module_name) module = _import_module(module_name)
if module: if module:
for group, opts in _list_opts(module): for group, opts in _list_opts(module):
opts_by_group.setdefault(group, []).append((module_name, opts)) opts_by_group.setdefault(group, []).append((module_name,
opts))
for pkg_name in pkg_names: for pkg_name in pkg_names:
mods = mods_by_pkg.get(pkg_name) mods = mods_by_pkg.get(pkg_name)
@@ -100,8 +102,8 @@ def generate(srcfiles):
opts_by_group.setdefault(group, []).append((mod_str, opts)) opts_by_group.setdefault(group, []).append((mod_str, opts))
print_group_opts('DEFAULT', opts_by_group.pop('DEFAULT', [])) print_group_opts('DEFAULT', opts_by_group.pop('DEFAULT', []))
for group, opts in opts_by_group.items(): for group in sorted(opts_by_group.keys()):
print_group_opts(group, opts) print_group_opts(group, opts_by_group[group])
def _import_module(mod_str): def _import_module(mod_str):
@@ -111,17 +113,17 @@ def _import_module(mod_str):
return sys.modules[mod_str[4:]] return sys.modules[mod_str[4:]]
else: else:
return importutils.import_module(mod_str) return importutils.import_module(mod_str)
except ImportError as ie: except Exception as e:
sys.stderr.write("%s\n" % str(ie)) sys.stderr.write("Error importing module %s: %s\n" % (mod_str, str(e)))
return None
except Exception:
return None return None
def _is_in_group(opt, group): def _is_in_group(opt, group):
"Check if opt is in group." "Check if opt is in group."
for key, value in group._opts.items(): for key, value in group._opts.items():
if value['opt'] == opt: # NOTE(llu): Temporary workaround for bug #1262148, wait until
# newly released oslo.config support '==' operator.
if not(value['opt'] != opt):
return True return True
return False return False
@@ -233,7 +235,7 @@ def _print_opt(opt):
if opt_default is None: if opt_default is None:
print('#%s=<None>' % opt_name) print('#%s=<None>' % opt_name)
elif opt_type == STROPT: elif opt_type == STROPT:
assert(isinstance(opt_default, basestring)) assert(isinstance(opt_default, six.string_types))
print('#%s=%s' % (opt_name, _sanitize_default(opt_name, print('#%s=%s' % (opt_name, _sanitize_default(opt_name,
opt_default))) opt_default)))
elif opt_type == BOOLOPT: elif opt_type == BOOLOPT:

6
tempest/openstack/common/excutils.py
View File

@@ -1,5 +1,3 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack Foundation. # Copyright 2011 OpenStack Foundation.
# Copyright 2012, Red Hat, Inc. # Copyright 2012, Red Hat, Inc.
# #
@@ -26,7 +24,7 @@ import traceback
import six import six
from tempest.openstack.common.gettextutils import _ # noqa from tempest.openstack.common.gettextutils import _
class save_and_reraise_exception(object): class save_and_reraise_exception(object):
@@ -44,7 +42,7 @@ class save_and_reraise_exception(object):
In some cases the caller may not want to re-raise the exception, and In some cases the caller may not want to re-raise the exception, and
for those circumstances this context provides a reraise flag that for those circumstances this context provides a reraise flag that
can be used to suppress the exception. For example: can be used to suppress the exception. For example::
except Exception: except Exception:
with save_and_reraise_exception() as ctxt: with save_and_reraise_exception() as ctxt:

4
tempest/openstack/common/fileutils.py
View File

@@ -1,5 +1,3 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack Foundation. # Copyright 2011 OpenStack Foundation.
# All Rights Reserved. # All Rights Reserved.
# #
@@ -22,7 +20,7 @@ import os
import tempfile import tempfile
from tempest.openstack.common import excutils from tempest.openstack.common import excutils
from tempest.openstack.common.gettextutils import _ # noqa from tempest.openstack.common.gettextutils import _
from tempest.openstack.common import log as logging from tempest.openstack.common import log as logging
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)

3
tempest/openstack/common/fixture/config.py
View File

@@ -1,4 +1,3 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# #
# Copyright 2013 Mirantis, Inc. # Copyright 2013 Mirantis, Inc.
# Copyright 2013 OpenStack Foundation # Copyright 2013 OpenStack Foundation
@@ -30,7 +29,7 @@ class Config(fixtures.Fixture):
the specified configuration option group. the specified configuration option group.
All overrides are automatically cleared at the end of the current All overrides are automatically cleared at the end of the current
test by the reset() method, which is registred by addCleanup(). test by the reset() method, which is registered by addCleanup().
""" """
def __init__(self, conf=cfg.CONF): def __init__(self, conf=cfg.CONF):

6
tempest/openstack/common/fixture/lockutils.py
View File

@@ -1,5 +1,3 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack Foundation. # Copyright 2011 OpenStack Foundation.
# All Rights Reserved. # All Rights Reserved.
# #
@@ -17,7 +15,7 @@
import fixtures import fixtures
from tempest.openstack.common.lockutils import lock from tempest.openstack.common import lockutils
class LockFixture(fixtures.Fixture): class LockFixture(fixtures.Fixture):
@@ -45,7 +43,7 @@ class LockFixture(fixtures.Fixture):
test method exits. (either by completing or raising an exception) test method exits. (either by completing or raising an exception)
""" """
def __init__(self, name, lock_file_prefix=None): def __init__(self, name, lock_file_prefix=None):
self.mgr = lock(name, lock_file_prefix, True) self.mgr = lockutils.lock(name, lock_file_prefix, True)
def setUp(self): def setUp(self):
super(LockFixture, self).setUp() super(LockFixture, self).setUp()

2
tempest/openstack/common/fixture/mockpatch.py
View File

@@ -1,5 +1,3 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010 United States Government as represented by the # Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration. # Administrator of the National Aeronautics and Space Administration.
# Copyright 2013 Hewlett-Packard Development Company, L.P. # Copyright 2013 Hewlett-Packard Development Company, L.P.

2
tempest/openstack/common/fixture/moxstubout.py
View File

@@ -1,5 +1,3 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010 United States Government as represented by the # Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration. # Administrator of the National Aeronautics and Space Administration.
# Copyright 2013 Hewlett-Packard Development Company, L.P. # Copyright 2013 Hewlett-Packard Development Company, L.P.

4
tempest/openstack/common/gettextutils.py
View File

@@ -1,5 +1,3 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2012 Red Hat, Inc. # Copyright 2012 Red Hat, Inc.
# Copyright 2013 IBM Corp. # Copyright 2013 IBM Corp.
# All Rights Reserved. # All Rights Reserved.
@@ -317,7 +315,7 @@ def get_available_languages(domain):
# NOTE(luisg): Babel <1.0 used a function called list(), which was # NOTE(luisg): Babel <1.0 used a function called list(), which was
# renamed to locale_identifiers() in >=1.0, the requirements master list # renamed to locale_identifiers() in >=1.0, the requirements master list
# requires >=0.9.6, uncapped, so defensively work with both. We can remove # requires >=0.9.6, uncapped, so defensively work with both. We can remove
# this check when the master list updates to >=1.0, and all projects udpate # this check when the master list updates to >=1.0, and update all projects
list_identifiers = (getattr(localedata, 'list', None) or list_identifiers = (getattr(localedata, 'list', None) or
getattr(localedata, 'locale_identifiers')) getattr(localedata, 'locale_identifiers'))
locale_identifiers = list_identifiers() locale_identifiers = list_identifiers()

2
tempest/openstack/common/importutils.py
View File

@@ -1,5 +1,3 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack Foundation. # Copyright 2011 OpenStack Foundation.
# All Rights Reserved. # All Rights Reserved.
# #

14
tempest/openstack/common/jsonutils.py
View File

@@ -1,5 +1,3 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010 United States Government as represented by the # Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration. # Administrator of the National Aeronautics and Space Administration.
# Copyright 2011 Justin Santa Barbara # Copyright 2011 Justin Santa Barbara
@@ -41,8 +39,12 @@ import json
try: try:
import xmlrpclib import xmlrpclib
except ImportError: except ImportError:
# NOTE(jd): xmlrpclib is not shipped with Python 3 # NOTE(jaypipes): xmlrpclib was renamed to xmlrpc.client in Python3
xmlrpclib = None # however the function and object call signatures
# remained the same. This whole try/except block should
# be removed and replaced with a call to six.moves once
# six 1.4.2 is released. See http://bit.ly/1bqrVzu
import xmlrpc.client as xmlrpclib
import six import six
@@ -124,14 +126,14 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
level=level, level=level,
max_depth=max_depth) max_depth=max_depth)
if isinstance(value, dict): if isinstance(value, dict):
return dict((k, recursive(v)) for k, v in value.iteritems()) return dict((k, recursive(v)) for k, v in six.iteritems(value))
elif isinstance(value, (list, tuple)): elif isinstance(value, (list, tuple)):
return [recursive(lv) for lv in value] return [recursive(lv) for lv in value]
# It's not clear why xmlrpclib created their own DateTime type, but # It's not clear why xmlrpclib created their own DateTime type, but
# for our purposes, make it a datetime type which is explicitly # for our purposes, make it a datetime type which is explicitly
# handled # handled
if xmlrpclib and isinstance(value, xmlrpclib.DateTime): if isinstance(value, xmlrpclib.DateTime):
value = datetime.datetime(*tuple(value.timetuple())[:6]) value = datetime.datetime(*tuple(value.timetuple())[:6])
if convert_datetime and isinstance(value, datetime.datetime): if convert_datetime and isinstance(value, datetime.datetime):

2
tempest/openstack/common/local.py
View File

@@ -1,5 +1,3 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack Foundation. # Copyright 2011 OpenStack Foundation.
# All Rights Reserved. # All Rights Reserved.
# #

4
tempest/openstack/common/lockutils.py
View File

@@ -1,5 +1,3 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack Foundation. # Copyright 2011 OpenStack Foundation.
# All Rights Reserved. # All Rights Reserved.
# #
@@ -31,7 +29,7 @@ import weakref
from oslo.config import cfg from oslo.config import cfg
from tempest.openstack.common import fileutils from tempest.openstack.common import fileutils
from tempest.openstack.common.gettextutils import _ # noqa from tempest.openstack.common.gettextutils import _
from tempest.openstack.common import local from tempest.openstack.common import local
from tempest.openstack.common import log as logging from tempest.openstack.common import log as logging

103
tempest/openstack/common/log.py
View File

@@ -1,5 +1,3 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack Foundation. # Copyright 2011 OpenStack Foundation.
# Copyright 2010 United States Government as represented by the # Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration. # Administrator of the National Aeronautics and Space Administration.
@@ -35,6 +33,7 @@ import logging
import logging.config import logging.config
import logging.handlers import logging.handlers
import os import os
import re
import sys import sys
import traceback import traceback
@@ -42,7 +41,7 @@ from oslo.config import cfg
import six import six
from six import moves from six import moves
from tempest.openstack.common.gettextutils import _ # noqa from tempest.openstack.common.gettextutils import _
from tempest.openstack.common import importutils from tempest.openstack.common import importutils
from tempest.openstack.common import jsonutils from tempest.openstack.common import jsonutils
from tempest.openstack.common import local from tempest.openstack.common import local
@@ -50,6 +49,24 @@ from tempest.openstack.common import local
_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
# NOTE(ldbragst): Let's build a list of regex objects using the list of
# _SANITIZE_KEYS we already have. This way, we only have to add the new key
# to the list of _SANITIZE_KEYS and we can generate regular expressions
# for XML and JSON automatically.
_SANITIZE_PATTERNS = []
_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
r'(<%(key)s>).*?(</%(key)s>)',
r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])']
for key in _SANITIZE_KEYS:
for pattern in _FORMAT_PATTERNS:
reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
_SANITIZE_PATTERNS.append(reg_ex)
common_cli_opts = [ common_cli_opts = [
cfg.BoolOpt('debug', cfg.BoolOpt('debug',
short='d', short='d',
@@ -64,11 +81,13 @@ common_cli_opts = [
] ]
logging_cli_opts = [ logging_cli_opts = [
cfg.StrOpt('log-config', cfg.StrOpt('log-config-append',
metavar='PATH', metavar='PATH',
help='If this option is specified, the logging configuration ' deprecated_name='log-config',
'file specified is used and overrides any other logging ' help='The name of logging configuration file. It does not '
'options specified. Please see the Python logging module ' 'disable existing loggers, but just appends specified '
'logging configuration to any other existing logging '
'options. Please see the Python logging module '
'documentation for details on logging configuration ' 'documentation for details on logging configuration '
'files.'), 'files.'),
cfg.StrOpt('log-format', cfg.StrOpt('log-format',
@@ -111,7 +130,7 @@ generic_log_opts = [
log_opts = [ log_opts = [
cfg.StrOpt('logging_context_format_string', cfg.StrOpt('logging_context_format_string',
default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
'%(name)s [%(request_id)s %(user)s %(tenant)s] ' '%(name)s [%(request_id)s %(user_identity)s] '
'%(instance)s%(message)s', '%(instance)s%(message)s',
help='format string to use for log messages with context'), help='format string to use for log messages with context'),
cfg.StrOpt('logging_default_format_string', cfg.StrOpt('logging_default_format_string',
@@ -127,12 +146,13 @@ log_opts = [
help='prefix each line of exception output with this format'), help='prefix each line of exception output with this format'),
cfg.ListOpt('default_log_levels', cfg.ListOpt('default_log_levels',
default=[ default=[
'amqp=WARN',
'amqplib=WARN', 'amqplib=WARN',
'sqlalchemy=WARN',
'boto=WARN', 'boto=WARN',
'qpid=WARN',
'sqlalchemy=WARN',
'suds=INFO', 'suds=INFO',
'keystone=INFO', 'iso8601=WARN',
'paramiko=INFO'
], ],
help='list of logger=LEVEL pairs'), help='list of logger=LEVEL pairs'),
cfg.BoolOpt('publish_errors', cfg.BoolOpt('publish_errors',
@@ -211,6 +231,40 @@ def _get_log_file_path(binary=None):
return None return None
def mask_password(message, secret="***"):
"""Replace password with 'secret' in message.
:param message: The string which includes security information.
:param secret: value with which to replace passwords.
:returns: The unicode value of message with the password fields masked.
For example:
>>> mask_password("'adminPass' : 'aaaaa'")
"'adminPass' : '***'"
>>> mask_password("'admin_pass' : 'aaaaa'")
"'admin_pass' : '***'"
>>> mask_password('"password" : "aaaaa"')
'"password" : "***"'
>>> mask_password("'original_password' : 'aaaaa'")
"'original_password' : '***'"
>>> mask_password("u'original_password' : u'aaaaa'")
"u'original_password' : u'***'"
"""
message = six.text_type(message)
# NOTE(ldbragst): Check to see if anything in message contains any key
# specified in _SANITIZE_KEYS, if not then just return the message since
# we don't have to mask any passwords.
if not any(key in message for key in _SANITIZE_KEYS):
return message
secret = r'\g<1>' + secret + r'\g<2>'
for pattern in _SANITIZE_PATTERNS:
message = re.sub(pattern, secret, message)
return message
class BaseLoggerAdapter(logging.LoggerAdapter): class BaseLoggerAdapter(logging.LoggerAdapter):
def audit(self, msg, *args, **kwargs): def audit(self, msg, *args, **kwargs):
@@ -278,10 +332,12 @@ class ContextAdapter(BaseLoggerAdapter):
elif instance_uuid: elif instance_uuid:
instance_extra = (CONF.instance_uuid_format instance_extra = (CONF.instance_uuid_format
% {'uuid': instance_uuid}) % {'uuid': instance_uuid})
extra.update({'instance': instance_extra}) extra['instance'] = instance_extra
extra.update({"project": self.project}) extra.setdefault('user_identity', kwargs.pop('user_identity', None))
extra.update({"version": self.version})
extra['project'] = self.project
extra['version'] = self.version
extra['extra'] = extra.copy() extra['extra'] = extra.copy()
return msg, kwargs return msg, kwargs
@@ -295,7 +351,7 @@ class JSONFormatter(logging.Formatter):
def formatException(self, ei, strip_newlines=True): def formatException(self, ei, strip_newlines=True):
lines = traceback.format_exception(*ei) lines = traceback.format_exception(*ei)
if strip_newlines: if strip_newlines:
lines = [itertools.ifilter( lines = [moves.filter(
lambda x: x, lambda x: x,
line.rstrip().splitlines()) for line in lines] line.rstrip().splitlines()) for line in lines]
lines = list(itertools.chain(*lines)) lines = list(itertools.chain(*lines))
@@ -333,10 +389,10 @@ class JSONFormatter(logging.Formatter):
def _create_logging_excepthook(product_name): def _create_logging_excepthook(product_name):
def logging_excepthook(type, value, tb): def logging_excepthook(exc_type, value, tb):
extra = {} extra = {}
if CONF.verbose: if CONF.verbose:
extra['exc_info'] = (type, value, tb) extra['exc_info'] = (exc_type, value, tb)
getLogger(product_name).critical(str(value), **extra) getLogger(product_name).critical(str(value), **extra)
return logging_excepthook return logging_excepthook
@@ -354,17 +410,18 @@ class LogConfigError(Exception):
err_msg=self.err_msg) err_msg=self.err_msg)
def _load_log_config(log_config): def _load_log_config(log_config_append):
try: try:
logging.config.fileConfig(log_config) logging.config.fileConfig(log_config_append,
disable_existing_loggers=False)
except moves.configparser.Error as exc: except moves.configparser.Error as exc:
raise LogConfigError(log_config, str(exc)) raise LogConfigError(log_config_append, str(exc))
def setup(product_name): def setup(product_name):
"""Setup logging.""" """Setup logging."""
if CONF.log_config: if CONF.log_config_append:
_load_log_config(CONF.log_config) _load_log_config(CONF.log_config_append)
else: else:
_setup_logging_from_conf() _setup_logging_from_conf()
sys.excepthook = _create_logging_excepthook(product_name) sys.excepthook = _create_logging_excepthook(product_name)
@@ -420,7 +477,7 @@ def _setup_logging_from_conf():
streamlog = ColorHandler() streamlog = ColorHandler()
log_root.addHandler(streamlog) log_root.addHandler(streamlog)
elif not CONF.log_file: elif not logpath:
# pass sys.stdout as a positional argument # pass sys.stdout as a positional argument
# python2.6 calls the argument strm, in 2.7 it's stream # python2.6 calls the argument strm, in 2.7 it's stream
streamlog = logging.StreamHandler(sys.stdout) streamlog = logging.StreamHandler(sys.stdout)

25
tempest/openstack/common/timeutils.py
View File

@@ -1,5 +1,3 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack Foundation. # Copyright 2011 OpenStack Foundation.
# All Rights Reserved. # All Rights Reserved.
# #
@@ -50,9 +48,9 @@ def parse_isotime(timestr):
try: try:
return iso8601.parse_date(timestr) return iso8601.parse_date(timestr)
except iso8601.ParseError as e: except iso8601.ParseError as e:
raise ValueError(unicode(e)) raise ValueError(six.text_type(e))
except TypeError as e: except TypeError as e:
raise ValueError(unicode(e)) raise ValueError(six.text_type(e))
def strtime(at=None, fmt=PERFECT_TIME_FORMAT): def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
@@ -79,6 +77,9 @@ def is_older_than(before, seconds):
"""Return True if before is older than seconds.""" """Return True if before is older than seconds."""
if isinstance(before, six.string_types): if isinstance(before, six.string_types):
before = parse_strtime(before).replace(tzinfo=None) before = parse_strtime(before).replace(tzinfo=None)
else:
before = before.replace(tzinfo=None)
return utcnow() - before > datetime.timedelta(seconds=seconds) return utcnow() - before > datetime.timedelta(seconds=seconds)
@@ -86,6 +87,9 @@ def is_newer_than(after, seconds):
"""Return True if after is newer than seconds.""" """Return True if after is newer than seconds."""
if isinstance(after, six.string_types): if isinstance(after, six.string_types):
after = parse_strtime(after).replace(tzinfo=None) after = parse_strtime(after).replace(tzinfo=None)
else:
after = after.replace(tzinfo=None)
return after - utcnow() > datetime.timedelta(seconds=seconds) return after - utcnow() > datetime.timedelta(seconds=seconds)
@@ -178,6 +182,15 @@ def delta_seconds(before, after):
datetime objects (as a float, to microsecond resolution). datetime objects (as a float, to microsecond resolution).
""" """
delta = after - before delta = after - before
return total_seconds(delta)
def total_seconds(delta):
"""Return the total seconds of datetime.timedelta object.
Compute total seconds of datetime.timedelta, datetime.timedelta
doesn't have method total_seconds in Python2.6, calculate it manually.
"""
try: try:
return delta.total_seconds() return delta.total_seconds()
except AttributeError: except AttributeError:
@@ -188,8 +201,8 @@ def delta_seconds(before, after):
def is_soon(dt, window): def is_soon(dt, window):
"""Determines if time is going to happen in the next window seconds. """Determines if time is going to happen in the next window seconds.
:params dt: the time :param dt: the time
:params window: minimum seconds to remain to consider the time not soon :param window: minimum seconds to remain to consider the time not soon
:return: True if expiration is within the given duration :return: True if expiration is within the given duration
""" """

6
tools/config/generate_sample.sh
View File

@@ -91,3 +91,9 @@ DEFAULT_MODULEPATH=tempest.openstack.common.config.generator
MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH} MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH}
OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample
python -m $MODULEPATH $FILES > $OUTPUTFILE python -m $MODULEPATH $FILES > $OUTPUTFILE
# Hook to allow projects to append custom config file snippets
CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null)
for CONCAT_FILE in $CONCAT_FILES; do
cat $CONCAT_FILE >> $OUTPUTFILE
done

2
tools/install_venv_common.py
View File

@@ -1,5 +1,3 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 OpenStack Foundation # Copyright 2013 OpenStack Foundation
# Copyright 2013 IBM Corp. # Copyright 2013 IBM Corp.
# #