Setup RPC and Notification hybrid messaging backends
Change-Id: Id60fb7d0f5220c3abc94c9a6478b3814d4622bb5
This commit is contained in:
parent
22851398e6
commit
05a09eb2cc
|
@ -1,38 +1,81 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
|
||||||
# devstack/plugin.sh
|
# Install and configure Apache Kafka. Since kafka is only used for
|
||||||
# Functions to install and configure Apache Kafka
|
# oslo.messaging notifications, setup a hybrid messaging backend.
|
||||||
|
# The RPC transport backend will be amqp:// and the
|
||||||
|
# Notification transport wil be kafka://
|
||||||
#
|
#
|
||||||
# Dependencies:
|
# Environment Configuration
|
||||||
#
|
# RPC_HOST - the host used to connect to the RPC messaging service.
|
||||||
# - ``functions`` file
|
# RPC_PORT - the port used to connect to the RPC messaging service.
|
||||||
#
|
# Defaults to 5672.
|
||||||
# ``stack.sh`` calls the entry points in this order:
|
# RPC_{USERNAME,PASSWORD} - for authentication with RPC messaging service.
|
||||||
#
|
# NOTIFY_HOST - the host used to connect to the Notification messaging service.
|
||||||
# - download_kafka
|
# NOTIFY_PORT - the port used to connect to the Notification messaging service.
|
||||||
# - install_kafka
|
# Defaults to 9092.
|
||||||
# - configure_kafka
|
# NOTIFY_{USERNAME,PASSWORD} - for authentication with Notification messaging
|
||||||
# - init_kafka
|
# service.
|
||||||
# - stop_kafka
|
|
||||||
# - cleanup_kafka
|
|
||||||
|
|
||||||
# Save trace setting
|
# Save trace setting
|
||||||
XTRACE=$(set +o | grep xtrace)
|
XTRACE=$(set +o | grep xtrace)
|
||||||
set +o xtrace
|
set +o xtrace
|
||||||
|
|
||||||
|
# builds rpc transport url string
|
||||||
|
function _get_rpc_transport_url {
|
||||||
|
if [ -z "$RPC_USERNAME" ]; then
|
||||||
|
echo "amqp://$RPC_HOST:${RPC_PORT}/"
|
||||||
|
else
|
||||||
|
echo "amqp://$RPC_USERNAME:$RPC_PASSWORD@$RPC_HOST:${RPC_PORT}/"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# builds notify transport url string
|
||||||
|
function _get_notify_transport_url {
|
||||||
|
if [ -z "$NOTIFY_USERNAME" ]; then
|
||||||
|
echo "kafka://$NOTIFY_HOST:${NOTIFY_PORT}/"
|
||||||
|
else
|
||||||
|
echo "kafka://$NOTIFY_USERNAME:$NOTIFY_PASSWORD@$NOTIFY_HOST:${NOTIFY_PORT}/"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
# Functions
|
# Functions
|
||||||
# ------------
|
# ------------
|
||||||
# download_kafka() - downloading kafka
|
# _download_kafka() - downloading kafka
|
||||||
function download_kafka {
|
function _download_kafka {
|
||||||
if [ ! -f ${FILES}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz ]; then
|
if [ ! -f ${FILES}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz ]; then
|
||||||
wget ${KAFKA_BASEURL}/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz \
|
wget ${KAFKA_BASEURL}/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz \
|
||||||
-O ${FILES}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz
|
-O ${FILES}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# install_kafka() - installing Kafka with Scala and Zookeeper
|
# install packages necessary for support of the oslo.messaging kafka
|
||||||
function install_kafka {
|
# driver
|
||||||
|
function _install_kafka_python {
|
||||||
|
# Install kafka client API
|
||||||
|
pip_install_gr kafka-python
|
||||||
|
}
|
||||||
|
|
||||||
|
# _install_kafka_backend() - installing Kafka with Scala and Zookeeper
|
||||||
|
function _install_kafka_backend {
|
||||||
|
echo_summary "Installing kafka service"
|
||||||
local scala_version=${SCALA_VERSION}.0
|
local scala_version=${SCALA_VERSION}.0
|
||||||
|
|
||||||
if is_ubuntu; then
|
if is_ubuntu; then
|
||||||
|
@ -57,7 +100,7 @@ function install_kafka {
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
download_kafka
|
_download_kafka
|
||||||
|
|
||||||
if [ ! -d ${FILES}/kafka_${SCALA_VERSION}-${KAFKA_VERSION} ]; then
|
if [ ! -d ${FILES}/kafka_${SCALA_VERSION}-${KAFKA_VERSION} ]; then
|
||||||
tar -xvzf ${FILES}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -C ${FILES}
|
tar -xvzf ${FILES}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -C ${FILES}
|
||||||
|
@ -66,58 +109,309 @@ function install_kafka {
|
||||||
mkdir -p ${KAFKA_DEST}
|
mkdir -p ${KAFKA_DEST}
|
||||||
mv ${FILES}/kafka_${SCALA_VERSION}-${KAFKA_VERSION} ${KAFKA_DEST}/kafka
|
mv ${FILES}/kafka_${SCALA_VERSION}-${KAFKA_VERSION} ${KAFKA_DEST}/kafka
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
_install_kafka_python
|
||||||
}
|
}
|
||||||
|
|
||||||
# init_kafka() - starting Kafka and Zookeeper processes
|
# _start_kafka_backend() - starting Kafka and Zookeeper processes
|
||||||
function init_kafka {
|
function _start_kafka_backend {
|
||||||
# start Zookeeper process before starting Kafka
|
# start Zookeeper process before starting Kafka
|
||||||
|
echo_summary "Initializing and starting kafka service"
|
||||||
${KAFKA_DEST}/kafka/bin/zookeeper-server-start.sh -daemon ${KAFKA_DEST}/kafka/config/zookeeper.properties
|
${KAFKA_DEST}/kafka/bin/zookeeper-server-start.sh -daemon ${KAFKA_DEST}/kafka/config/zookeeper.properties
|
||||||
${KAFKA_DEST}/kafka/bin/kafka-server-start.sh -daemon ${KAFKA_DEST}/kafka/config/server.properties
|
${KAFKA_DEST}/kafka/bin/kafka-server-start.sh -daemon ${KAFKA_DEST}/kafka/config/server.properties
|
||||||
}
|
}
|
||||||
|
|
||||||
# configure_kafka() - configuring Kafka service
|
# _configure_kafka() - configuring Kafka service
|
||||||
function configure_kafka {
|
function _configure_kafka {
|
||||||
# currently a no op
|
# currently a no op
|
||||||
:
|
:
|
||||||
}
|
}
|
||||||
|
|
||||||
# stop_kafka() - stopping Kafka process
|
# _stop_kafka() - stopping Kafka process
|
||||||
function stop_kafka {
|
function _stop_kafka {
|
||||||
|
echo_summary "Shut down kafka service"
|
||||||
${KAFKA_DEST}/kafka/bin/kafka-server-stop.sh
|
${KAFKA_DEST}/kafka/bin/kafka-server-stop.sh
|
||||||
${KAFKA_DEST}/kafka/bin/zookeeper-server-stop.sh
|
${KAFKA_DEST}/kafka/bin/zookeeper-server-stop.sh
|
||||||
}
|
}
|
||||||
|
|
||||||
# cleanup_kafka() - removing Kafka files
|
# _cleanup_kafka() - removing Kafka files
|
||||||
# make sure this function is called only after calling stop_kafka() function
|
# make sure this function is called only after calling stop_kafka() function
|
||||||
function cleanup_kafka {
|
function _cleanup_kafka {
|
||||||
|
echo_summary "Clean up kafka service"
|
||||||
rm ${FILES}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz
|
rm ${FILES}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz
|
||||||
rm -rf ${KAFKA_DEST}/kafka
|
rm -rf ${KAFKA_DEST}/kafka
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Set up the various configuration files used by the qpid-dispatch-router (qdr)
|
||||||
|
function _configure_qdr {
|
||||||
|
|
||||||
|
# the location of the configuration is /etc/qpid-dispatch
|
||||||
|
local qdr_conf_file
|
||||||
|
if [ -e /etc/qpid-dispatch/qdrouterd.conf ]; then
|
||||||
|
qdr_conf_file=/etc/qpid-dispatch/qdrouterd.conf
|
||||||
|
else
|
||||||
|
exit_distro_not_supported "qdrouterd.conf file not found!"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ensure that the qpid-dispatch-router service can read its config
|
||||||
|
sudo chmod o+r $qdr_conf_file
|
||||||
|
|
||||||
|
# qdouterd.conf file customization for devstack deployment
|
||||||
|
# Define attributes related to the AMQP container
|
||||||
|
# Create stand alone router
|
||||||
|
cat <<EOF | sudo tee $qdr_conf_file
|
||||||
|
router {
|
||||||
|
mode: standalone
|
||||||
|
id: Router.A
|
||||||
|
workerThreads: 4
|
||||||
|
saslConfigPath: /etc/sasl2
|
||||||
|
saslConfigName: qdrouterd
|
||||||
|
debugDump: /opt/stack/amqp1
|
||||||
|
}
|
||||||
|
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# Create a listener for incoming connect to the router
|
||||||
|
cat <<EOF | sudo tee --append $qdr_conf_file
|
||||||
|
listener {
|
||||||
|
addr: 0.0.0.0
|
||||||
|
port: ${RPC_PORT}
|
||||||
|
role: normal
|
||||||
|
EOF
|
||||||
|
if [ -z "$RPC_USERNAME" ]; then
|
||||||
|
#no user configured, so disable authentication
|
||||||
|
cat <<EOF | sudo tee --append $qdr_conf_file
|
||||||
|
authenticatePeer: no
|
||||||
|
}
|
||||||
|
|
||||||
|
EOF
|
||||||
|
else
|
||||||
|
# configure to use PLAIN authentication
|
||||||
|
if [ -z "$RPC_PASSWORD" ]; then
|
||||||
|
read_password RPC_PASSWORD "ENTER A PASSWORD FOR QPID DISPATCH USER $RPC_USERNAME"
|
||||||
|
fi
|
||||||
|
cat <<EOF | sudo tee --append $qdr_conf_file
|
||||||
|
authenticatePeer: yes
|
||||||
|
}
|
||||||
|
|
||||||
|
EOF
|
||||||
|
# Add user to SASL database
|
||||||
|
local sasl_conf_file=/etc/sasl2/qdrouterd.conf
|
||||||
|
cat <<EOF | sudo tee $sasl_conf_file
|
||||||
|
pwcheck_method: auxprop
|
||||||
|
auxprop_plugin: sasldb
|
||||||
|
sasldb_path: /var/lib/qdrouterd/qdrouterd.sasldb
|
||||||
|
mech_list: PLAIN
|
||||||
|
sql_select: dummy select
|
||||||
|
EOF
|
||||||
|
local sasl_db
|
||||||
|
sasl_db=`sudo grep sasldb_path $sasl_conf_file | cut -f 2 -d ":" | tr -d [:blank:]`
|
||||||
|
if [ ! -e $sasl_db ]; then
|
||||||
|
sudo mkdir -p -m 755 `dirname $sasl_db`
|
||||||
|
fi
|
||||||
|
echo $RPC_PASSWORD | sudo saslpasswd2 -c -p -f $sasl_db $RPC_USERNAME
|
||||||
|
sudo chmod o+r $sasl_db
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Create fixed address prefixes
|
||||||
|
cat <<EOF | sudo tee --append $qdr_conf_file
|
||||||
|
address {
|
||||||
|
prefix: unicast
|
||||||
|
distribution: closest
|
||||||
|
}
|
||||||
|
|
||||||
|
address {
|
||||||
|
prefix: exclusive
|
||||||
|
distribution: closest
|
||||||
|
}
|
||||||
|
|
||||||
|
address {
|
||||||
|
prefix: broadcast
|
||||||
|
distribution: multicast
|
||||||
|
}
|
||||||
|
|
||||||
|
address {
|
||||||
|
prefix: openstack.org/om/rpc/multicast
|
||||||
|
distribution: multicast
|
||||||
|
}
|
||||||
|
|
||||||
|
address {
|
||||||
|
prefix: openstack.org/om/rpc/unicast
|
||||||
|
distribution: closest
|
||||||
|
}
|
||||||
|
|
||||||
|
address {
|
||||||
|
prefix: openstack.org/om/rpc/anycast
|
||||||
|
distribution: balanced
|
||||||
|
}
|
||||||
|
|
||||||
|
address {
|
||||||
|
prefix: openstack.org/om/notify/multicast
|
||||||
|
distribution: multicast
|
||||||
|
}
|
||||||
|
|
||||||
|
address {
|
||||||
|
prefix: openstack.org/om/notify/unicast
|
||||||
|
distribution: closest
|
||||||
|
}
|
||||||
|
|
||||||
|
address {
|
||||||
|
prefix: openstack.org/om/notify/anycast
|
||||||
|
distribution: balanced
|
||||||
|
}
|
||||||
|
|
||||||
|
EOF
|
||||||
|
|
||||||
|
local log_file=$LOGDIR/qdrouterd.log
|
||||||
|
|
||||||
|
sudo touch $log_file
|
||||||
|
sudo chmod a+rw $log_file # qdrouterd user can write to it
|
||||||
|
|
||||||
|
# Create log file configuration
|
||||||
|
cat <<EOF | sudo tee --append $qdr_conf_file
|
||||||
|
log {
|
||||||
|
module: DEFAULT
|
||||||
|
enable: info+
|
||||||
|
output: $log_file
|
||||||
|
}
|
||||||
|
|
||||||
|
EOF
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
# install packages necessary for support of the oslo.messaging AMQP
|
||||||
|
# 1.0 driver
|
||||||
|
function _install_pyngus {
|
||||||
|
# Install pyngus client API
|
||||||
|
if is_fedora; then
|
||||||
|
# TODO(kgiusti) due to a bug in the way pip installs wheels,
|
||||||
|
# do not let pip install the proton python bindings as it will
|
||||||
|
# put them in the wrong path:
|
||||||
|
# https://github.com/pypa/pip/issues/2940
|
||||||
|
install_package python-qpid-proton
|
||||||
|
elif is_ubuntu; then
|
||||||
|
# ditto
|
||||||
|
install_package python-qpid-proton
|
||||||
|
fi
|
||||||
|
pip_install_gr pyngus
|
||||||
|
}
|
||||||
|
|
||||||
|
# install and configure the amqp1 backend
|
||||||
|
# dispatch-router for hybrid deployment with kakfa
|
||||||
|
function _install_qdr_backend {
|
||||||
|
echo_summary "Installing amqp1 service qdrouterd"
|
||||||
|
if is_fedora; then
|
||||||
|
# expects epel is already added to the yum repos
|
||||||
|
install_package cyrus-sasl-lib
|
||||||
|
install_package cyrus-sasl-plain
|
||||||
|
install_package qpid-dispatch-router
|
||||||
|
elif is_ubuntu; then
|
||||||
|
install_package sasl2-bin
|
||||||
|
# qdrouterd and proton only available via the qpid PPA
|
||||||
|
sudo add-apt-repository -y ppa:qpid/released
|
||||||
|
#sudo apt-get update
|
||||||
|
REPOS_UPDATED=False
|
||||||
|
update_package_repo
|
||||||
|
install_package qdrouterd
|
||||||
|
else
|
||||||
|
exit_distro_not_supported "amqp1 qdrouterd installation"
|
||||||
|
fi
|
||||||
|
|
||||||
|
_install_pyngus
|
||||||
|
_configure_qdr
|
||||||
|
}
|
||||||
|
|
||||||
|
function _start_qdr_backend {
|
||||||
|
echo_summary "Starting qdrouterd backend"
|
||||||
|
# restart, since qdrouterd may already be running
|
||||||
|
restart_service qdrouterd
|
||||||
|
}
|
||||||
|
|
||||||
|
function _stop_qdr_backend {
|
||||||
|
echo_summary "Stopping qdrouterd backend"
|
||||||
|
stop_service qdrouterd
|
||||||
|
}
|
||||||
|
|
||||||
|
# remove packages used by oslo.messaging AMQP 1.0 driver
|
||||||
|
function _remove_pyngus {
|
||||||
|
# TODO(ansmith) no way to pip uninstall?
|
||||||
|
# pip_install_gr pyngus
|
||||||
|
:
|
||||||
|
}
|
||||||
|
|
||||||
|
function _cleanup_qdr_backend {
|
||||||
|
if is_fedora; then
|
||||||
|
uninstall_package qpid-dispatch-router
|
||||||
|
# TODO(ansmith) can we pull these, or will that break other
|
||||||
|
# packages that depend on them?
|
||||||
|
|
||||||
|
# install_package cyrus_sasl_lib
|
||||||
|
# install_package cyrus_sasl_plain
|
||||||
|
elif is_ubuntu; then
|
||||||
|
uninstall_package qdrouterd
|
||||||
|
# install_package sasl2-bin
|
||||||
|
else
|
||||||
|
exit_distro_not_supported "amqp1 qdrouterd installation"
|
||||||
|
fi
|
||||||
|
|
||||||
|
_remove_pyngus
|
||||||
|
}
|
||||||
|
|
||||||
|
if is_service_enabled kafka; then
|
||||||
|
|
||||||
|
# Note: this is the only tricky part about out of tree
|
||||||
|
# oslo.messaging plugins, you must overwrite the functions
|
||||||
|
# so that the correct settings files are made.
|
||||||
|
function iniset_rpc_backend {
|
||||||
|
local package=$1
|
||||||
|
local file=$2
|
||||||
|
local section=${3:-DEFAULT}
|
||||||
|
iniset $file $section transport_url $(_get_rpc_transport_url)
|
||||||
|
iniset $file oslo_messaging_notifications transport_url $(_get_notify_transport_url)
|
||||||
|
}
|
||||||
|
function get_transport_url {
|
||||||
|
# TODO (ansmith) introduce separate get_*_transport calls in devstak
|
||||||
|
_get_rpc_transport_url $@
|
||||||
|
}
|
||||||
|
function get_notification_url {
|
||||||
|
_get_notify_transport_url $@
|
||||||
|
}
|
||||||
|
export -f iniset_rpc_backend
|
||||||
|
export -f get_transport_url
|
||||||
|
export -f get_notification_url
|
||||||
|
fi
|
||||||
|
|
||||||
# check for kafka service
|
# check for kafka service
|
||||||
if is_service_enabled devstack-plugin-kafka; then
|
if is_service_enabled kafka; then
|
||||||
if [[ "$1" == "source" ]]; then
|
if [[ "$1" == "source" ]]; then
|
||||||
# Initial source
|
# Initial source
|
||||||
source $TOP_DIR/lib/kafka
|
source $TOP_DIR/lib/kafka
|
||||||
|
|
||||||
elif [[ "$1" == "stack" && "$2" == "install" ]]; then
|
elif [[ "$1" == "stack" && "$2" == "pre-install" ]]; then
|
||||||
# Perform installation of service source
|
# nothing needed here
|
||||||
echo_summary "Installing kafka"
|
:
|
||||||
install_kafka
|
|
||||||
|
|
||||||
echo_summary "Initializing kafka"
|
elif [[ "$1" == "stack" && "$2" == "install" ]]; then
|
||||||
init_kafka
|
# Install and configure the messaging services
|
||||||
|
_install_kafka_backend
|
||||||
|
_install_qdr_backend
|
||||||
|
|
||||||
|
elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
|
||||||
|
# Start the messaging service processes, this happens before
|
||||||
|
# any services start
|
||||||
|
_start_kafka_backend
|
||||||
|
_start_qdr_backend
|
||||||
|
|
||||||
elif [[ "$1" == "unstack" ]]; then
|
elif [[ "$1" == "unstack" ]]; then
|
||||||
# Shut down kafka services
|
# Shut down messaging services
|
||||||
echo_summary "Shut down kafka service"
|
_stop_kafka
|
||||||
stop_kafka
|
_stop_qdr
|
||||||
|
|
||||||
elif [[ "$1" == "clean" ]]; then
|
elif [[ "$1" == "clean" ]]; then
|
||||||
# Remove state and transient data
|
# Remove state and transient data
|
||||||
# Remember clean.sh first calls unstack.sh
|
# Remember clean.sh first calls unstack.sh
|
||||||
echo_summary "Clean up kafka service"
|
_cleanup_kafka
|
||||||
cleanup_kafka
|
_cleanup_qdr
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
|
@ -6,9 +6,19 @@ enable_service kafka
|
||||||
KAFKA_DEST=${KAFKA_DEST:-/opt/stack/devstack-plugin-kafka}
|
KAFKA_DEST=${KAFKA_DEST:-/opt/stack/devstack-plugin-kafka}
|
||||||
|
|
||||||
# Specify Kafka version
|
# Specify Kafka version
|
||||||
KAFKA_VERSION=${KAFKA_VERSION:-0.9.0.1}
|
KAFKA_VERSION=${KAFKA_VERSION:-0.10.2.1}
|
||||||
KAFKA_BASEURL=${KAFKA_BASEURL:-http://www.us.apache.org/dist/kafka}
|
KAFKA_BASEURL=${KAFKA_BASEURL:-http://www.us.apache.org/dist/kafka}
|
||||||
|
|
||||||
# Specify Scala version
|
# Specify Scala version
|
||||||
SCALA_VERSION=${SCALA_VERSION:-2.10}
|
SCALA_VERSION=${SCALA_VERSION:-2.12}
|
||||||
SCALA_BASEURL=${SCALA_BASEURL:-http://www.scala-lang.org/files/archive}
|
SCALA_BASEURL=${SCALA_BASEURL:-http://www.scala-lang.org/files/archive}
|
||||||
|
|
||||||
|
# RPC
|
||||||
|
RPC_HOST=${RPC_HOST:-$SERVICE_HOST}
|
||||||
|
RPC_PORT=${RPC_PORT:-5672}
|
||||||
|
|
||||||
|
# Notify
|
||||||
|
NOTIFY_HOST=${NOTIFY_HOST:-$SERVICE_HOST}
|
||||||
|
NOTIFY_PORT=${NOTIFY_PORT:-9092}
|
||||||
|
|
||||||
|
disable_service rabbit
|
||||||
|
|
Loading…
Reference in New Issue