Add devstack support

Add some scripts that devstack needs.

Change-Id: I8b9ad477ba0d2d9d581f90c67d5b46b32a4c66a3
This commit is contained in:
baiziyu 2021-03-12 19:42:18 +08:00
parent f7cc58c6ac
commit 4d59bc461c
24 changed files with 363 additions and 56 deletions

4
.gitignore vendored
View File

@ -7,6 +7,10 @@
.stestr/
*.DS_Store
devstack/files/*.deb
devstack/files/*.deb.*
venv/
dist
build/*
venus.egg-info/

View File

@ -3,4 +3,6 @@ include ChangeLog
exclude .gitignore
exclude .gitreview
recursive-include venus/db/sqlalchemy/migrate_repo *
global-exclude *.pyc

View File

@ -0,0 +1,9 @@
<filter *.log>
@type record_transformer
<record>
Hostname ${hostname}
Logger openstack.${tag_parts[4]}
programname ${tag_parts[5]}
</record>
</filter>

View File

@ -0,0 +1,4 @@
<match *.log>
@type rewrite_tag_filter
capitalize_regex_backreference yes
</match>

View File

@ -0,0 +1,10 @@
<filter apache_access>
@type parser
reserve_data true
format grok
key_name message
grok_pattern \[%{HTTPDATE:Timestamp}\] "(?:%{WORD:http_method} %{NOTSPACE:http_url}(?: HTTP/%{NUMBER:http_version})?|%{DATA:rawrequest})" %{NUMBER:http_status} (?:\d+|-)
time_key Timestamp
time_format %d/%b/%Y:%H:%M:%S %z
keep_time_key true
</filter>

View File

@ -0,0 +1,10 @@
<filter wsgi_access>
@type parser
reserve_data true
format grok
key_name message
grok_pattern %{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:Timestamp}\] "(?:%{WORD:http_method} %{NOTSPACE:http_url}(?: HTTP/%{NUMBER:http_version})?|%{DATA:rawrequest})" %{NUMBER:http_status} (?:%{NUMBER:http_bytes}|-) (?:%{NUMBER:http_response_time_us}|-) %{QS:referrer} %{QS:agent}
time_key Timestamp
time_format %d/%b/%Y:%H:%M:%S %z
keep_time_key true
</filter>

View File

@ -0,0 +1,15 @@
<source>
@type tail
path /var/log/kolla/neutron/*.log
pos_file /var/run/td-agent/devstack-openstack_neutron.pos
tag devstack
format multiline
format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
time_key Timestamp
keep_time_key true
time_format %F %T.%L
ignore_repeated_permission_error true
enable_watch_timer false
</source>

View File

@ -0,0 +1,15 @@
<source>
@type tail
path /var/log/kolla/vitrage/*.log
pos_file /var/run/td-agent/devstack-openstack_vitrage.pos
tag devstack
format multiline
format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
time_key Timestamp
keep_time_key true
time_format %F %T.%L
ignore_repeated_permission_error true
enable_watch_timer false
</source>

View File

@ -0,0 +1,15 @@
<source>
@type tail
path /var/log/kolla/keystone/*.log
pos_file /var/run/td-agent/devstack-openstack_keystone.pos
tag devstack
format multiline
format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
time_key Timestamp
keep_time_key true
time_format %F %T.%L
ignore_repeated_permission_error true
enable_watch_timer false
</source>

View File

@ -0,0 +1,15 @@
<source>
@type tail
path /var/log/kolla/cinder/*.log
pos_file /var/run/td-agent/devstack-openstack_cinder.pos
tag devstack
format multiline
format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(\S+ req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
format1 /^(?<Timestamp>\S+ \S+) (?<Pid>\d+) (?<log_level>\S+) (?<python_module>\S+) (\[(?<global_id>\S+) (req-(?<request_id>\S+) (?<user_id>\S+) (?<tenant_id>\S+) (?<domain_id>\S+) (?<user_domain>\S+) (?<project_domain>\S+)|-)\])? (?<Payload>.*)?$/
time_key Timestamp
keep_time_key true
time_format %F %T.%L
ignore_repeated_permission_error true
enable_watch_timer false
</source>

View File

@ -0,0 +1,9 @@
<source>
@type tail
path /var/log/bootstrap.log,/var/log/btmp,/var/log/syslog
pos_file /var/run/td-agent/devstack-syslog.pos
tag syslog.*
source_hostname_key Hostname
format /^(?<Payload>.*)$/
enable_watch_timer false
</source>

View File

@ -0,0 +1,18 @@
<match syslog.**>
@type copy
<store>
type elasticsearch
host localhost
port 9200
logstash_format true
logstash_prefix slog
buffer_type memory
buffer_chunk_limit 8m
buffer_queue_limit 8
buffer_queue_full_action drop_oldest_chunk
flush_interval 30s
num_threads 4
reconnect_on_error true
resurrect_after 5s
</store>
</match>

View File

@ -0,0 +1,18 @@
<match devstack>
@type copy
<store>
@type elasticsearch
host localhost
port 9200
logstash_format true
logstash_prefix flog
buffer_type memory
buffer_chunk_limit 8m
buffer_queue_limit 8
buffer_queue_full_action drop_oldest_chunk
flush_interval 30s
num_threads 4
reconnect_on_error true
resurrect_after 5s
</store>
</match>

View File

@ -0,0 +1,5 @@
@include input/*.conf
@include filter/*.conf
#@include format/*.conf
@include output/*.conf

163
devstack/plugin.sh Normal file
View File

@ -0,0 +1,163 @@
# plugin.sh - DevStack plugin.sh dispatch script venus
# Support potential entry-points console scripts in VENV or not
if [[ ${USE_VENV} == True ]]; then
PROJECT_VENV["venus"]=${VENUS_DIR}.venv
VENUS_BIN_DIR=${PROJECT_VENV["venus"]}/bin
else
VENUS_BIN_DIR=$(get_python_exec_prefix)
fi
FILES=$VENUS_DIR/devstack/files
function install_venus() {
setup_develop "$VENUS_DIR" openstack
install_fluentd
install_elastic_search
}
function init_venus() {
venus_create_accounts
recreate_database venus utf8
sudo $VENUS_BIN_DIR/venus-manage --config-file=${VENUS_CONF} db sync
}
function configure_venus() {
# Create venus conf directory
sudo install -d -o $STACK_USER -m 755 $VENUS_CONF_DIR
# Copy init conf file
sudo cp -R $VENUS_DIR/etc/venus/* $VENUS_CONF_DIR
iniset $VENUS_CONF keystone_authtoken memcached_servers localhost:11211
iniset $VENUS_CONF keystone_authtoken username venus
iniset $VENUS_CONF keystone_authtoken password "$ADMIN_PASSWORD"
iniset $VENUS_CONF keystone_authtoken auth_url "http://$HOST_IP/identity"
iniset $VENUS_CONF DEFAULT my_ip "$HOST_IP"
iniset $VENUS_CONF DEFAULT osapi_venus_listen_port 10010
iniset $VENUS_CONF database connection mysql+pymysql://root:"$DATABASE_PASSWORD"@localhost:3306/venus?charset=utf8
iniset $VENUS_CONF elasticsearch url http://localhost:9200
}
function start_venus() {
run_process venus-api "$VENUS_BIN_DIR/venus-api"
}
function install_elastic_search() {
echo_summary "install elastic search"
local FLUENTD_SERVICE="elasticsearch.service"
if [[ is_ubuntu ]]; then
install_package openjdk-8-jdk
ES_VERSION=${ES_VERSION:-5.6.16}
ES_DOWNLOAD_URL=${ES_DOWNLOAD_URL:-https://artifacts.elastic.co/downloads/elasticsearch}
ES_DOWNLOAD_FILE="elasticsearch-$ES_VERSION.deb"
if [[ ! -f $FILES/$ES_DOWNLOAD_FILE ]]; then
sudo wget --progress=dot:giga -t 2 -c $ES_DOWNLOAD_URL/$ES_DOWNLOAD_FILE -O $FILES/$ES_DOWNLOAD_FILE
if [[ $? -ne 0 ]]; then
die "$ES_DOWNLOAD_FILE could not be downloaded"
fi
fi
sudo dpkg -i $FILES/$ES_DOWNLOAD_FILE
$SYSTEMCTL daemon-reload
$SYSTEMCTL enable $FLUENTD_SERVICE
$SYSTEMCTL start $FLUENTD_SERVICE
else
exit_distro_not_supported "install elastic search"
fi
}
function install_fluentd() {
echo_summary "install fluentd"
if [[ is_ubuntu ]]; then
FLUENTD_VERSION=${FLUENTD_VERSION:-4.1.0-1_amd64}
FLUENTD_DOWNLOAD_URL=${FLUENTD_DOWNLOAD_URL:-http://packages.treasuredata.com.s3.amazonaws.com/4/ubuntu/bionic/pool/contrib/t/td-agent}
FLUENTD_DOWNLOAD_FILE="td-agent_$FLUENTD_VERSION.deb"
if [[ ! -f $FILES/$FLUENTD_DOWNLOAD_FILE ]]; then
sudo wget --progress=dot:giga -t 2 -c $FLUENTD_DOWNLOAD_URL/$FLUENTD_DOWNLOAD_FILE -O $FILES/$FLUENTD_DOWNLOAD_FILE
if [[ $? -ne 0 ]]; then
die "$FLUENTD_DOWNLOAD_FILE could not be downloaded"
fi
fi
sudo dpkg -i $FILES/$FLUENTD_DOWNLOAD_FILE
else
exit_distro_not_supported "install fluentd"
fi
# Create log dir
VENUS_LOG_DIR="/var/log/kolla"
sudo install -d -o $STACK_USER -m 777 $VENUS_LOG_DIR
# Copy fluentd conf
sudo cp -R $VENUS_DIR/devstack/fluentd-conf/* /etc/td-agent
$SYSTEMCTL restart td-agent
}
function venus_create_accounts() {
create_service_user "venus"
}
function uninstall_elastic_search() {
local ELASTIC_SEARCH_SERVICE="elasticsearch.service"
$SYSTEMCTL stop $ELASTIC_SEARCH_SERVICE
$SYSTEMCTL disable $ELASTIC_SEARCH_SERVICE
$SYSTEMCTL daemon-reload
sudo dpkg -r elasticsearch
}
function uninstall_fluentd() {
local FLUENTD_SERVICE="td-agent.service"
$SYSTEMCTL stop $FLUENTD_SERVICE
$SYSTEMCTL disable $FLUENTD_SERVICE
$SYSTEMCTL daemon-reload
sudo dpkg -r td-agent
}
# check for service enabled
if is_service_enabled venus-api; then
if [[ "$1" == "stack" && "$2" == "pre-install" ]]; then
# Set up system services
echo_summary "Configuring system services venus"
echo_summary "Welcome to Venus!"
elif [[ "$1" == "stack" && "$2" == "install" ]]; then
# Perform installation of service source
echo_summary "Installing venus"
install_venus
elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
# Configure after the other layer 1 and 2 services have been configured
echo_summary "Configuring venus"
configure_venus
elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
# Initialize and start the venus service
echo_summary "Initializing venus"
init_venus
start_venus
fi
if [[ "$1" == "unstack" ]]; then
# Shut down venus services
# no-op
:
fi
if [[ "$1" == "clean" ]]; then
# Remove state and transient data
# Remember clean.sh first calls unstack.sh
# no-op
uninstall_elastic_search
uninstall_fluentd
sudo rm -rf $VENUS_CONF_DIR
fi
fi

9
devstack/settings Normal file
View File

@ -0,0 +1,9 @@
# settings file for venus
# API Service
enable_service venus-api
# Default directories
VENUS_DIR=$DEST/venus
VENUS_CONF_DIR=/etc/venus
VENUS_CONF=$VENUS_CONF_DIR/venus.conf

View File

@ -1,37 +1,27 @@
[keystone_authtoken]
memcached_servers = 100.2.30.241:11211,100.2.30.242:11211,100.2.30.243:11211signing_dir = /var/cache/venus
signing_dir = /var/cache/venus
cafile = /opt/stack/data/ca-bundle.pem
project_domain_name = default
project_domain_name = Default
project_name = service
user_domain_name = default
password = dTa74mdF29CyGLQvH8RCKAhFPlRd1zHtp2Ai4NGw
user_domain_name = Default
password = secret
username = venus
auth_uri = http://100.2.28.240:5000
auth_url = http://100.2.28.240:35357
project_domain_id = default
user_domain_id = default
auth_url = http://0.0.0.0/identity
auth_type = password
[DEFAULT]
transport_url = rabbit://openstack:R8axM8sde8Dq5tV1PcDHmDRPLsA9fBLpXrGQccfE@100.2.30.243:5672
my_ip = 100.2.30.243
transport_url = rabbit://openstack:secret@0.0.0.0:5672
my_ip = 0.0.0.0
periodic_interval = 60
rootwrap_config = /etc/venus/rootwrap.conf
api_paste_config = /etc/venus/api-paste.ini
log_dir = /var/log/kolla/venus/
use_syslog = False
debug = True
use_stderr = True
auth_strategy = keystone
os_region_name = RegionOne
osapi_venus_listen = 100.2.30.243
osapi_venus_listen_port = 8686
osapi_venus_listen_port = 10010
osapi_venus_workers = 1
[database]
connection = mysql+pymysql://root:Irpzw6tic9ezyUEh4c0JnT0kK7U1oKqbRPRIfkwW@100.2.28.72:3306/venus?charset=utf8
connection = mysql+pymysql://root:secret@localhost:3306/venus?charset=utf8
[elasticsearch]
url = http://100.2.28.30:9200
username = admin
password = DlR7Y4vcPPbwbOCHYO8f8zG9VtwnLrd1t5R1A3B9
url = http://localhost:9200

View File

@ -24,8 +24,6 @@ packages =
venus
[entry_points]
venus.database.migration_backend =
sqlalchemy = venus.db.sqlalchemy.migration
wsgi_scripts =
venus-api-wsgi = venus.cmd.api:main
console_scripts =
venus-manage = venus.cmd.manage:main
venus-api = venus.cmd.api:main

View File

@ -15,7 +15,6 @@
"""Starter script for Venus OS API."""
import eventlet
import os
import sys
from oslo_log import log as logging
@ -36,10 +35,10 @@ def main():
objects.register_all()
CONF(sys.argv[1:], project='venus',
version=version.version_string())
logdir = CONF.log_dir
is_exits = os.path.exists(logdir)
if not is_exits:
os.makedirs(logdir)
# logdir = CONF.log_dir
# is_exits = os.path.exists(logdir)
# if not is_exits:
# os.makedirs(logdir)
logging.setup(CONF, "venus")
utils.monkey_patch()

View File

@ -345,10 +345,10 @@ def main():
try:
CONF(sys.argv[1:], project='venus',
version=version.version_string())
logdir = CONF.log_dir
is_exits = os.path.exists(logdir)
if not is_exits:
os.makedirs(logdir)
# logdir = CONF.log_dir
# is_exits = os.path.exists(logdir)
# if not is_exits:
# os.makedirs(logdir)
logging.setup(CONF, "venus")
except cfg.ConfigDirNotFoundError as details:
print(_("Invalid directory: %s") % details)

View File

@ -23,7 +23,7 @@ service_opts = [
default=60,
help='Range, in seconds, to randomly delay when starting the'
' periodic task scheduler to reduce stampeding.'
' (Disable by setting to 0)'),
' (Disable by settings to 0)'),
cfg.StrOpt('osapi_venus_listen',
default="0.0.0.0",
help='IP address on which OpenStack Venus API listens'),

View File

@ -134,7 +134,8 @@ global_opts = [
default=True,
help='If False, closes the client socket connection '
'explicitly. Setting it to True to maintain backward '
'compatibility. Recommended setting is set it to False.'),
'compatibility. Recommended settings is set it '
'to False.'),
cfg.BoolOpt('fatal_exception_format_errors',
default=False,
help='Make exception message format errors fatal.'),

View File

@ -17,42 +17,41 @@
import os
import threading
from oslo_db import options
from stevedore import driver
from venus.conf import CONF
from venus.db.sqlalchemy import api as db_api
from oslo_config import cfg
from oslo_db import options as db_options
from oslo_db.sqlalchemy.migration import db_sync as sync
from oslo_db.sqlalchemy import session as db_session
INIT_VERSION = 000
_IMPL = None
_LOCK = threading.Lock()
options.set_defaults(CONF)
db_options.set_defaults(cfg.CONF)
MIGRATE_REPO_PATH = os.path.join(
os.path.abspath(os.path.dirname(__file__)),
'sqlalchemy',
'migrate_repo',
'sqlalchemy/migrate_repo',
)
def get_backend():
def get_engine():
print(cfg.CONF)
global _IMPL
if _IMPL is None:
with _LOCK:
if _IMPL is None:
_IMPL = driver.DriverManager(
"venus.database.migration_backend",
CONF.database.backend).driver
return _IMPL
_IMPL = db_session.EngineFacade(
cfg.CONF.database.connection,
**dict(cfg.CONF.database)
)
return _IMPL.get_engine()
def db_sync(version=None, init_version=INIT_VERSION, engine=None):
"""Migrate the database to `version` or the most recent version."""
if engine is None:
engine = db_api.get_engine()
return get_backend().db_sync(engine=engine,
engine = get_engine()
return sync(engine=engine,
abs_path=MIGRATE_REPO_PATH,
version=version,
init_version=init_version)

View File

@ -1 +0,0 @@
My fist commit.