Browse Source

Setup RPC and Notification hybrid messaging backends

Change-Id: Id60fb7d0f5220c3abc94c9a6478b3814d4622bb5
Andrew Smith 1 year ago
parent
commit
05a09eb2cc
3 changed files with 343 additions and 39 deletions
  1. 330
    36
      devstack/plugin.sh
  2. 12
    2
      devstack/settings
  3. 1
    1
      tox.ini

+ 330
- 36
devstack/plugin.sh View File

@@ -1,38 +1,81 @@
1 1
 #!/bin/bash
2
-
3
-# devstack/plugin.sh
4
-# Functions to install and configure Apache Kafka
5 2
 #
6
-# Dependencies:
3
+#    Licensed to the Apache Software Foundation (ASF) under one
4
+#    or more contributor license agreements.  See the NOTICE file
5
+#    distributed with this work for additional information
6
+#    regarding copyright ownership.
7
+#
8
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
9
+#    not use this file except in compliance with the License. You may obtain
10
+#    a copy of the License at
7 11
 #
8
-#   - ``functions`` file
12
+#         http://www.apache.org/licenses/LICENSE-2.0
9 13
 #
10
-# ``stack.sh`` calls the entry points in this order:
14
+#    Unless required by applicable law or agreed to in writing, software
15
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17
+#    License for the specific language governing permissions and limitations
18
+#    under the License.
11 19
 #
12
-#   - download_kafka
13
-#   - install_kafka
14
-#   - configure_kafka
15
-#   - init_kafka
16
-#   - stop_kafka
17
-#   - cleanup_kafka
20
+
21
+# Install and configure Apache Kafka. Since kafka is only used for
22
+# oslo.messaging notifications, setup a hybrid messaging backend.
23
+# The RPC transport backend will be amqp:// and the
24
+# Notification transport wil be kafka://
25
+#
26
+# Environment Configuration
27
+# RPC_HOST - the host used to connect to the RPC messaging service.
28
+# RPC_PORT - the port used to connect to the RPC messaging service.
29
+#     Defaults to 5672.
30
+# RPC_{USERNAME,PASSWORD} - for authentication with RPC messaging service.
31
+# NOTIFY_HOST - the host used to connect to the Notification messaging service.
32
+# NOTIFY_PORT - the port used to connect to the Notification messaging service.
33
+#     Defaults to 9092.
34
+# NOTIFY_{USERNAME,PASSWORD} - for authentication with Notification messaging
35
+#     service.
18 36
 
19 37
 # Save trace setting
20 38
 XTRACE=$(set +o | grep xtrace)
21 39
 set +o xtrace
22 40
 
41
+# builds rpc transport url string
42
+function _get_rpc_transport_url {
43
+    if [ -z "$RPC_USERNAME" ]; then
44
+        echo "amqp://$RPC_HOST:${RPC_PORT}/"
45
+    else
46
+        echo "amqp://$RPC_USERNAME:$RPC_PASSWORD@$RPC_HOST:${RPC_PORT}/"
47
+    fi
48
+}
49
+
50
+# builds notify transport url string
51
+function _get_notify_transport_url {
52
+    if [ -z "$NOTIFY_USERNAME" ]; then
53
+        echo "kafka://$NOTIFY_HOST:${NOTIFY_PORT}/"
54
+    else
55
+        echo "kafka://$NOTIFY_USERNAME:$NOTIFY_PASSWORD@$NOTIFY_HOST:${NOTIFY_PORT}/"
56
+    fi
57
+}
58
+
23 59
 # Functions
24 60
 # ------------
25
-# download_kafka() - downloading kafka
26
-function download_kafka {
61
+# _download_kafka() - downloading kafka
62
+function _download_kafka {
27 63
     if [ ! -f ${FILES}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz ]; then
28 64
         wget ${KAFKA_BASEURL}/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz \
29 65
             -O ${FILES}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz
30 66
     fi
31 67
 }
32 68
 
33
-# install_kafka() - installing Kafka with Scala and Zookeeper
34
-function install_kafka {
69
+# install packages necessary for support of the oslo.messaging kafka
70
+# driver
71
+function _install_kafka_python {
72
+    # Install kafka client API
73
+    pip_install_gr kafka-python
74
+}
35 75
 
76
+# _install_kafka_backend() - installing Kafka with Scala and Zookeeper
77
+function _install_kafka_backend {
78
+    echo_summary "Installing kafka service"
36 79
     local scala_version=${SCALA_VERSION}.0
37 80
 
38 81
     if is_ubuntu; then
@@ -57,7 +100,7 @@ function install_kafka {
57 100
         fi
58 101
     fi
59 102
 
60
-    download_kafka
103
+    _download_kafka
61 104
 
62 105
     if [ ! -d ${FILES}/kafka_${SCALA_VERSION}-${KAFKA_VERSION} ]; then
63 106
         tar -xvzf ${FILES}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -C ${FILES}
@@ -66,58 +109,309 @@ function install_kafka {
66 109
         mkdir -p ${KAFKA_DEST}
67 110
         mv ${FILES}/kafka_${SCALA_VERSION}-${KAFKA_VERSION} ${KAFKA_DEST}/kafka
68 111
     fi
112
+
113
+    _install_kafka_python
69 114
 }
70 115
 
71
-# init_kafka() - starting Kafka and Zookeeper processes
72
-function init_kafka {
116
+# _start_kafka_backend() - starting Kafka and Zookeeper processes
117
+function _start_kafka_backend {
73 118
     # start Zookeeper process before starting Kafka
119
+    echo_summary "Initializing and starting kafka service"
74 120
     ${KAFKA_DEST}/kafka/bin/zookeeper-server-start.sh -daemon ${KAFKA_DEST}/kafka/config/zookeeper.properties
75 121
     ${KAFKA_DEST}/kafka/bin/kafka-server-start.sh -daemon ${KAFKA_DEST}/kafka/config/server.properties
76 122
 }
77 123
 
78
-# configure_kafka() - configuring Kafka service
79
-function configure_kafka {
124
+# _configure_kafka() - configuring Kafka service
125
+function _configure_kafka {
80 126
     # currently a no op
81 127
     :
82 128
 }
83 129
 
84
-# stop_kafka() - stopping Kafka process
85
-function stop_kafka {
130
+# _stop_kafka() - stopping Kafka process
131
+function _stop_kafka {
132
+    echo_summary "Shut down kafka service"
86 133
     ${KAFKA_DEST}/kafka/bin/kafka-server-stop.sh
87 134
     ${KAFKA_DEST}/kafka/bin/zookeeper-server-stop.sh
88 135
 }
89 136
 
90
-# cleanup_kafka() - removing Kafka files
137
+# _cleanup_kafka() - removing Kafka files
91 138
 # make sure this function is called only after calling stop_kafka() function
92
-function cleanup_kafka {
139
+function _cleanup_kafka {
140
+    echo_summary "Clean up kafka service"
93 141
     rm ${FILES}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz
94 142
     rm -rf ${KAFKA_DEST}/kafka
95 143
 }
96 144
 
145
+# Set up the various configuration files used by the qpid-dispatch-router (qdr)
146
+function _configure_qdr {
147
+
148
+    # the location of the configuration is /etc/qpid-dispatch
149
+    local qdr_conf_file
150
+    if [ -e /etc/qpid-dispatch/qdrouterd.conf ]; then
151
+        qdr_conf_file=/etc/qpid-dispatch/qdrouterd.conf
152
+    else
153
+        exit_distro_not_supported "qdrouterd.conf file not found!"
154
+    fi
155
+
156
+    # ensure that the qpid-dispatch-router service can read its config
157
+    sudo chmod o+r $qdr_conf_file
158
+
159
+    # qdouterd.conf file customization for devstack deployment
160
+    # Define attributes related to the AMQP container
161
+    # Create stand alone router
162
+    cat <<EOF | sudo tee $qdr_conf_file
163
+router {
164
+    mode: standalone
165
+    id: Router.A
166
+    workerThreads: 4
167
+    saslConfigPath: /etc/sasl2
168
+    saslConfigName: qdrouterd
169
+    debugDump: /opt/stack/amqp1
170
+}
171
+
172
+EOF
173
+
174
+    # Create a listener for incoming connect to the router
175
+    cat <<EOF | sudo tee --append $qdr_conf_file
176
+listener {
177
+    addr: 0.0.0.0
178
+    port: ${RPC_PORT}
179
+    role: normal
180
+EOF
181
+    if [ -z "$RPC_USERNAME" ]; then
182
+        #no user configured, so disable authentication
183
+        cat <<EOF | sudo tee --append $qdr_conf_file
184
+    authenticatePeer: no
185
+}
186
+
187
+EOF
188
+    else
189
+        # configure to use PLAIN authentication
190
+        if [ -z "$RPC_PASSWORD" ]; then
191
+            read_password RPC_PASSWORD "ENTER A PASSWORD FOR QPID DISPATCH USER $RPC_USERNAME"
192
+        fi
193
+        cat <<EOF | sudo tee --append $qdr_conf_file
194
+    authenticatePeer: yes
195
+}
196
+
197
+EOF
198
+        # Add user to SASL database
199
+        local sasl_conf_file=/etc/sasl2/qdrouterd.conf
200
+        cat <<EOF | sudo tee $sasl_conf_file
201
+pwcheck_method: auxprop
202
+auxprop_plugin: sasldb
203
+sasldb_path: /var/lib/qdrouterd/qdrouterd.sasldb
204
+mech_list: PLAIN
205
+sql_select: dummy select
206
+EOF
207
+        local sasl_db
208
+        sasl_db=`sudo grep sasldb_path $sasl_conf_file | cut -f 2 -d ":" | tr -d [:blank:]`
209
+        if [ ! -e $sasl_db ]; then
210
+            sudo mkdir -p -m 755 `dirname $sasl_db`
211
+        fi
212
+        echo $RPC_PASSWORD | sudo saslpasswd2 -c -p -f $sasl_db $RPC_USERNAME
213
+        sudo chmod o+r $sasl_db
214
+    fi
215
+
216
+    # Create fixed address prefixes
217
+    cat <<EOF | sudo tee --append $qdr_conf_file
218
+address {
219
+    prefix: unicast
220
+    distribution: closest
221
+}
222
+
223
+address {
224
+    prefix: exclusive
225
+    distribution: closest
226
+}
227
+
228
+address {
229
+    prefix: broadcast
230
+    distribution: multicast
231
+}
232
+
233
+address {
234
+    prefix: openstack.org/om/rpc/multicast
235
+    distribution: multicast
236
+}
237
+
238
+address {
239
+    prefix: openstack.org/om/rpc/unicast
240
+    distribution: closest
241
+}
242
+
243
+address {
244
+    prefix: openstack.org/om/rpc/anycast
245
+    distribution: balanced
246
+}
247
+
248
+address {
249
+    prefix: openstack.org/om/notify/multicast
250
+    distribution: multicast
251
+}
252
+
253
+address {
254
+    prefix: openstack.org/om/notify/unicast
255
+    distribution: closest
256
+}
257
+
258
+address {
259
+    prefix: openstack.org/om/notify/anycast
260
+    distribution: balanced
261
+}
262
+
263
+EOF
264
+
265
+    local log_file=$LOGDIR/qdrouterd.log
266
+
267
+    sudo touch $log_file
268
+    sudo chmod a+rw $log_file  # qdrouterd user can write to it
269
+
270
+    # Create log file configuration
271
+    cat <<EOF | sudo tee --append $qdr_conf_file
272
+log {
273
+    module: DEFAULT
274
+    enable: info+
275
+    output: $log_file
276
+}
277
+
278
+EOF
279
+
280
+}
281
+
282
+# install packages necessary for support of the oslo.messaging AMQP
283
+# 1.0 driver
284
+function _install_pyngus {
285
+    # Install pyngus client API
286
+    if is_fedora; then
287
+        # TODO(kgiusti) due to a bug in the way pip installs wheels,
288
+        # do not let pip install the proton python bindings as it will
289
+        # put them in the wrong path:
290
+        # https://github.com/pypa/pip/issues/2940
291
+        install_package python-qpid-proton
292
+    elif is_ubuntu; then
293
+        # ditto
294
+        install_package python-qpid-proton
295
+    fi
296
+    pip_install_gr pyngus
297
+}
298
+
299
+# install and configure the amqp1 backend
300
+# dispatch-router for hybrid deployment with kakfa
301
+function _install_qdr_backend {
302
+    echo_summary "Installing amqp1 service qdrouterd"
303
+    if is_fedora; then
304
+        # expects epel is already added to the yum repos
305
+        install_package cyrus-sasl-lib
306
+        install_package cyrus-sasl-plain
307
+        install_package qpid-dispatch-router
308
+    elif is_ubuntu; then
309
+        install_package sasl2-bin
310
+        # qdrouterd and proton only available via the qpid PPA
311
+        sudo add-apt-repository -y ppa:qpid/released
312
+        #sudo apt-get update
313
+        REPOS_UPDATED=False
314
+        update_package_repo
315
+        install_package qdrouterd
316
+    else
317
+        exit_distro_not_supported "amqp1 qdrouterd installation"
318
+    fi
319
+
320
+    _install_pyngus
321
+    _configure_qdr
322
+}
323
+
324
+function _start_qdr_backend {
325
+    echo_summary "Starting qdrouterd backend"
326
+    # restart, since qdrouterd may already be running
327
+    restart_service qdrouterd
328
+}
329
+
330
+function _stop_qdr_backend {
331
+    echo_summary "Stopping qdrouterd backend"
332
+    stop_service qdrouterd
333
+}
334
+
335
+# remove packages used by oslo.messaging AMQP 1.0 driver
336
+function _remove_pyngus {
337
+    # TODO(ansmith) no way to pip uninstall?
338
+    # pip_install_gr pyngus
339
+    :
340
+}
341
+
342
+function _cleanup_qdr_backend {
343
+    if is_fedora; then
344
+        uninstall_package qpid-dispatch-router
345
+        # TODO(ansmith) can we pull these, or will that break other
346
+        # packages that depend on them?
347
+
348
+        # install_package cyrus_sasl_lib
349
+        # install_package cyrus_sasl_plain
350
+    elif is_ubuntu; then
351
+        uninstall_package qdrouterd
352
+        # install_package sasl2-bin
353
+    else
354
+        exit_distro_not_supported "amqp1 qdrouterd installation"
355
+    fi
356
+
357
+    _remove_pyngus
358
+}
359
+
360
+if is_service_enabled kafka; then
361
+
362
+    # Note: this is the only tricky part about out of tree
363
+    # oslo.messaging plugins, you must overwrite the functions
364
+    # so that the correct settings files are made.
365
+    function iniset_rpc_backend {
366
+        local package=$1
367
+        local file=$2
368
+        local section=${3:-DEFAULT}
369
+        iniset $file $section transport_url $(_get_rpc_transport_url)
370
+        iniset $file oslo_messaging_notifications transport_url $(_get_notify_transport_url)
371
+    }
372
+    function get_transport_url {
373
+        # TODO (ansmith) introduce separate get_*_transport calls in devstak
374
+        _get_rpc_transport_url $@
375
+    }
376
+    function get_notification_url {
377
+        _get_notify_transport_url $@
378
+    }
379
+    export -f iniset_rpc_backend
380
+    export -f get_transport_url
381
+    export -f get_notification_url
382
+fi
383
+
97 384
 # check for kafka service
98
-if is_service_enabled devstack-plugin-kafka; then
385
+if is_service_enabled kafka; then
99 386
     if [[ "$1" == "source" ]]; then
100 387
         # Initial source
101 388
         source $TOP_DIR/lib/kafka
102 389
 
390
+    elif [[ "$1" == "stack" && "$2" == "pre-install" ]]; then
391
+        # nothing needed here
392
+        :
393
+
103 394
     elif [[ "$1" == "stack" && "$2" == "install" ]]; then
104
-        # Perform installation of service source
105
-        echo_summary "Installing kafka"
106
-        install_kafka
395
+        # Install and configure the messaging services
396
+        _install_kafka_backend
397
+        _install_qdr_backend
107 398
 
108
-        echo_summary "Initializing kafka"
109
-        init_kafka
399
+    elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
400
+        # Start the messaging service processes, this happens before
401
+        # any services start
402
+        _start_kafka_backend
403
+        _start_qdr_backend
110 404
 
111 405
     elif [[ "$1" == "unstack" ]]; then
112
-        # Shut down kafka services
113
-        echo_summary "Shut down kafka service"
114
-        stop_kafka
406
+        # Shut down messaging services
407
+        _stop_kafka
408
+        _stop_qdr
115 409
 
116 410
     elif [[ "$1" == "clean" ]]; then
117 411
         # Remove state and transient data
118 412
         # Remember clean.sh first calls unstack.sh
119
-        echo_summary "Clean up kafka service"
120
-        cleanup_kafka
413
+        _cleanup_kafka
414
+        _cleanup_qdr
121 415
     fi
122 416
 fi
123 417
 

+ 12
- 2
devstack/settings View File

@@ -6,9 +6,19 @@ enable_service kafka
6 6
 KAFKA_DEST=${KAFKA_DEST:-/opt/stack/devstack-plugin-kafka}
7 7
 
8 8
 # Specify Kafka version
9
-KAFKA_VERSION=${KAFKA_VERSION:-0.9.0.1}
9
+KAFKA_VERSION=${KAFKA_VERSION:-0.10.2.1}
10 10
 KAFKA_BASEURL=${KAFKA_BASEURL:-http://www.us.apache.org/dist/kafka}
11 11
 
12 12
 # Specify Scala version
13
-SCALA_VERSION=${SCALA_VERSION:-2.10}
13
+SCALA_VERSION=${SCALA_VERSION:-2.12}
14 14
 SCALA_BASEURL=${SCALA_BASEURL:-http://www.scala-lang.org/files/archive}
15
+
16
+# RPC
17
+RPC_HOST=${RPC_HOST:-$SERVICE_HOST}
18
+RPC_PORT=${RPC_PORT:-5672}
19
+
20
+# Notify
21
+NOTIFY_HOST=${NOTIFY_HOST:-$SERVICE_HOST}
22
+NOTIFY_PORT=${NOTIFY_PORT:-9092}
23
+
24
+disable_service rabbit

+ 1
- 1
tox.ini View File

@@ -22,4 +22,4 @@ commands = bash -c "find {toxinidir}          \
22 22
           -name functions\* -or               \
23 23
           -wholename \*/lib/\*                \
24 24
          \)                                   \
25
-         -print0 | xargs -0 bashate -v"
25
+         -print0 | xargs -0 bashate -v -iE006"

Loading…
Cancel
Save