OpenStack Search (Searchlight)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

418 lines
15 KiB

  1. # Install and start **Searchlight** service
  2. # To enable Searchlight services, add the following to localrc
  3. # enable_plugin searchlight https://opendev.org/openstack/searchlight
  4. # enable_service searchlight-api
  5. # enable_service searchlight-listener
  6. # stack.sh
  7. # ---------
  8. # install_searchlight
  9. # configure_searchlight
  10. # init_searchlight
  11. # start_searchlight
  12. # stop_searchlight
  13. # cleanup_searchlight
  14. # Save trace setting
  15. XTRACE=$(set +o | grep xtrace)
  16. set +o xtrace
  17. # Defaults
  18. # --------
  19. # Set up default repos
  20. SEARCHLIGHT_REPO=${SEARCHLIGHT_REPO:-${GIT_BASE}/openstack/searchlight.git}
  21. SEARCHLIGHT_BRANCH=${SEARCHLIGHT_BRANCH:-master}
  22. # Set up default paths
  23. SEARCHLIGHT_BIN_DIR=$(get_python_exec_prefix)
  24. SEARCHLIGHT_DIR=$DEST/searchlight
  25. SEARCHLIGHT_CONF_DIR=/etc/searchlight
  26. SEARCHLIGHT_STATE_PATH=${SEARCHLIGHT_STATE_PATH:=$DATA_DIR/searchlight}
  27. SEARCHLIGHT_CONF=$SEARCHLIGHT_CONF_DIR/searchlight.conf
  28. SEARCHLIGHT_LOG_DIR=/var/log/searchlight
  29. SEARCHLIGHT_AUTH_CACHE_DIR=${SEARCHLIGHT_AUTH_CACHE_DIR:-/var/cache/searchlight}
  30. SEARCHLIGHT_APIPASTE_CONF=$SEARCHLIGHT_CONF_DIR/api-paste.ini
  31. SEARCHLIGHT_UWSGI_CONF=$SEARCHLIGHT_CONF_DIR/searchlight-uwsgi.ini
  32. SEARCHLIGHT_UWSGI_APP=$SEARCHLIGHT_BIN_DIR/searchlight-api-wsgi
  33. if is_service_enabled tls-proxy; then
  34. SEARCHLIGHT_SERVICE_PROTOCOL="https"
  35. fi
  36. # Public IP/Port Settings
  37. SEARCHLIGHT_SERVICE_PROTOCOL=${SEARCHLIGHT_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
  38. SEARCHLIGHT_SERVICE_HOST=${SEARCHLIGHT_SERVICE_HOST:-$SERVICE_HOST}
  39. SEARCHLIGHT_SERVICE_PORT=${SEARCHLIGHT_SERVICE_PORT:-9393}
  40. SEARCHLIGHT_SERVICE_PORT_INT=${SEARCHLIGHT_SERVICE_PORT_INT:-19393}
  41. ELASTICSEARCH_VERSION=${ELASTICSEARCH_VERSION:-2.3.4}
  42. # Base URL for ElasticSearch 5.x and 6.x
  43. ELASTICSEARCH_BASEURL=https://artifacts.elastic.co/downloads/elasticsearch
  44. # Base URL for ElasticSearch 2.x
  45. ELASTICSEARCH_BASEURL_LEGACY=https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution
  46. # Helper Functions
  47. # ----------------
  48. function setup_colorized_logging_searchlight {
  49. local conf_file=$1
  50. local conf_section=$2
  51. local project_var=${3:-"project_name"}
  52. local user_var=${4:-"user_name"}
  53. setup_colorized_logging $conf_file $conf_section $project_var $user_var
  54. # Override the logging_context_format_string value chosen by
  55. # setup_colorized_logging.
  56. iniset $conf_file $conf_section logging_context_format_string "%(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [%(request_id)s %(user_identity)s%(color)s] %(instance)s%(color)s%(message)s"
  57. }
  58. # DevStack Plugin
  59. # ---------------
  60. # cleanup_searchlight - Remove residual data files, anything left over from previous
  61. # runs that a clean run would need to clean up
  62. function cleanup_searchlight {
  63. _stop_elasticsearch
  64. sudo rm -rf $SEARCHLIGHT_STATE_PATH $SEARCHLIGHT_AUTH_CACHE_DIR
  65. sudo rm -f $(apache_site_config_for searchlight_api)
  66. }
  67. # configure_searchlight - Set config files, create data dirs, etc
  68. function configure_searchlight {
  69. [ ! -d $SEARCHLIGHT_CONF_DIR ] && sudo mkdir -m 755 -p $SEARCHLIGHT_CONF_DIR
  70. sudo chown $STACK_USER $SEARCHLIGHT_CONF_DIR
  71. [ ! -d $SEARCHLIGHT_LOG_DIR ] && sudo mkdir -m 755 -p $SEARCHLIGHT_LOG_DIR
  72. sudo chown $STACK_USER $SEARCHLIGHT_LOG_DIR
  73. # (Re)create ``searchlight.conf``
  74. rm -f $SEARCHLIGHT_CONF
  75. # General Configuration
  76. iniset_rpc_backend searchlight $SEARCHLIGHT_CONF DEFAULT
  77. iniset $SEARCHLIGHT_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
  78. iniset $SEARCHLIGHT_CONF DEFAULT state_path $SEARCHLIGHT_STATE_PATH
  79. # API Configuration
  80. sudo cp $SEARCHLIGHT_DIR/etc/api-paste.ini $SEARCHLIGHT_APIPASTE_CONF
  81. iniset $SEARCHLIGHT_CONF api public_endpoint $SEARCHLIGHT_SERVICE_PROTOCOL://$SEARCHLIGHT_SERVICE_HOST/search
  82. # OpenStack users
  83. iniset $SEARCHLIGHT_CONF service_credentials auth_type password
  84. iniset $SEARCHLIGHT_CONF service_credentials username searchlight
  85. iniset $SEARCHLIGHT_CONF service_credentials user_domain_id default
  86. iniset $SEARCHLIGHT_CONF service_credentials project_domain_id default
  87. iniset $SEARCHLIGHT_CONF service_credentials password $SERVICE_PASSWORD
  88. iniset $SEARCHLIGHT_CONF service_credentials project_name $SERVICE_PROJECT_NAME
  89. iniset $SEARCHLIGHT_CONF service_credentials auth_url $KEYSTONE_SERVICE_URI
  90. iniset $SEARCHLIGHT_CONF service_credentials os_region_name $REGION_NAME
  91. # Keystone Middleware
  92. iniset $SEARCHLIGHT_CONF paste_deploy flavor keystone
  93. configure_auth_token_middleware $SEARCHLIGHT_CONF searchlight $SEARCHLIGHT_AUTH_CACHE_DIR
  94. # Oslo Concurrency
  95. iniset $SEARCHLIGHT_CONF oslo_concurrency lock_path "$SEARCHLIGHT_STATE_PATH"
  96. # TLS Proxy Configuration
  97. if is_service_enabled tls-proxy; then
  98. # Set the service port for a proxy to take the original
  99. iniset $SEARCHLIGHT_CONF api bind_port $SEARCHLIGHT_SERVICE_PORT_INT
  100. else
  101. iniset $SEARCHLIGHT_CONF api bind_port $SEARCHLIGHT_SERVICE_PORT
  102. fi
  103. # Logging Configuration
  104. if [ "$SYSLOG" != "False" ]; then
  105. iniset $SEARCHLIGHT_CONF DEFAULT use_syslog True
  106. fi
  107. # Format logging
  108. if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then
  109. setup_colorized_logging_searchlight $SEARCHLIGHT_CONF DEFAULT "tenant" "user"
  110. fi
  111. # Plugin config - disable designate by default since it's not typically installed
  112. iniset $SEARCHLIGHT_CONF resource_plugin:os_designate_zone enabled False
  113. iniset $SEARCHLIGHT_CONF resource_plugin:os_designate_recordset enabled False
  114. # Plugin config - disable ironic by default since it's not typically installed
  115. iniset $SEARCHLIGHT_CONF resource_plugin:os_ironic_chassis enabled False
  116. iniset $SEARCHLIGHT_CONF resource_plugin:os_ironic_chassis notifications_topics_exchanges ironic_versioned_notifications,ironic
  117. iniset $SEARCHLIGHT_CONF resource_plugin:os_ironic_node enabled False
  118. iniset $SEARCHLIGHT_CONF resource_plugin:os_ironic_node notifications_topics_exchanges ironic_versioned_notifications,ironic
  119. iniset $SEARCHLIGHT_CONF resource_plugin:os_ironic_port enabled False
  120. iniset $SEARCHLIGHT_CONF resource_plugin:os_ironic_port notifications_topics_exchanges ironic_versioned_notifications,ironic
  121. # Plugin config - enable versioned notifications for flavor
  122. iniset $SEARCHLIGHT_CONF resource_plugin:os_nova_flavor enabled True
  123. iniset $SEARCHLIGHT_CONF resource_plugin:os_nova_flavor notifications_topics_exchanges versioned_notifications,nova
  124. iniset $SEARCHLIGHT_CONF resource_plugin:os_nova_server enabled True
  125. iniset $SEARCHLIGHT_CONF resource_plugin:os_nova_server notifications_topics_exchanges versioned_notifications,nova
  126. iniset $SEARCHLIGHT_CONF resource_plugin:os_nova_server use_versioned_notifications True
  127. # Plugin config - disable swift by default since it's not typically installed
  128. iniset $SEARCHLIGHT_CONF resource_plugin:os_swift_account enabled False
  129. iniset $SEARCHLIGHT_CONF resource_plugin:os_swift_container enabled False
  130. iniset $SEARCHLIGHT_CONF resource_plugin:os_swift_object enabled False
  131. # uWSGI configuration
  132. write_uwsgi_config "$SEARCHLIGHT_UWSGI_CONF" "$SEARCHLIGHT_UWSGI_APP" "/search"
  133. }
  134. # create_searchlight_accounts - Set up common required searchlight accounts
  135. # Tenant User Roles
  136. # ------------------------------------------------------------------
  137. # service searchlight admin # if enabled
  138. function create_searchlight_accounts {
  139. if [[ "$ENABLED_SERVICES" =~ "searchlight-" ]]; then
  140. create_service_user "searchlight" "admin"
  141. if is_service_enabled searchlight-api; then
  142. get_or_create_service "searchlight" "search" "Searchlight Service"
  143. get_or_create_endpoint "search" \
  144. "$REGION_NAME" \
  145. "$SEARCHLIGHT_SERVICE_PROTOCOL://$SEARCHLIGHT_SERVICE_HOST/search" \
  146. "$SEARCHLIGHT_SERVICE_PROTOCOL://$SEARCHLIGHT_SERVICE_HOST/search" \
  147. "$SEARCHLIGHT_SERVICE_PROTOCOL://$SEARCHLIGHT_SERVICE_HOST/search"
  148. fi
  149. fi
  150. }
  151. # init_searchlight - Initialize etc.
  152. function init_searchlight {
  153. # Create cache dir
  154. sudo mkdir -p $SEARCHLIGHT_AUTH_CACHE_DIR
  155. sudo chown $STACK_USER $SEARCHLIGHT_AUTH_CACHE_DIR
  156. rm -f $SEARCHLIGHT_AUTH_CACHE_DIR/*
  157. _start_elasticsearch
  158. $SEARCHLIGHT_BIN_DIR/searchlight-manage --config-file $SEARCHLIGHT_CONF index sync --force
  159. }
  160. # Install Searchlight's requirements
  161. # See https://elasticsearch-py.readthedocs.io/en/master/#compatibility
  162. function _setup_searchlight_dev {
  163. setup_develop $SEARCHLIGHT_DIR
  164. if [[ $ELASTICSEARCH_VERSION =~ ^5 ]]; then
  165. echo "Installing python elasticsearch for ES 5.x"
  166. $REQUIREMENTS_DIR/.venv/bin/edit-constraints $REQUIREMENTS_DIR/upper-constraints.txt elasticsearch
  167. pip_install -U -r $SEARCHLIGHT_DIR/elasticsearch5.txt
  168. elif [[ $ELASTICSEARCH_VERSION =~ ^6 ]]; then
  169. echo "WARNING - Searchlight is not tested with ES 6.x!!!"
  170. # echo "Installing python elasticsearch for ES 6.x"
  171. # $REQUIREMENTS_DIR/.venv/bin/edit-constraints $REQUIREMENTS_DIR/upper-constraints.txt eleasticsearch
  172. # pip install -c $REQUIREMENTS_DIR/upper-constraints.txt -U -r $SEARCHLIGHT_DIR/elasticsearch6.txt
  173. fi
  174. }
  175. # install_searchlight - Collect source and prepare
  176. function install_searchlight {
  177. git_clone $SEARCHLIGHT_REPO $SEARCHLIGHT_DIR $SEARCHLIGHT_BRANCH
  178. _setup_searchlight_dev
  179. _download_elasticsearch
  180. _install_elasticsearch
  181. pip_install uwsgi
  182. }
  183. # install_searchlightclient - Collect source and prepare
  184. function install_searchlightclient {
  185. git_clone $SEARCHLIGHTCLIENT_REPO $SEARCHLIGHTCLIENT_DIR $SEARCHLIGHTCLIENT_BRANCH
  186. setup_develop $SEARCHLIGHTCLIENT_DIR
  187. }
  188. # start_searchlight - Start running processes, including screen
  189. function start_searchlight {
  190. if is_service_enabled searchlight-api; then
  191. run_process searchlight-api "$SEARCHLIGHT_BIN_DIR/uwsgi --ini $SEARCHLIGHT_UWSGI_CONF"
  192. fi
  193. if is_service_enabled searchlight-listener; then
  194. run_process searchlight-listener "$SEARCHLIGHT_BIN_DIR/searchlight-listener --config-file $SEARCHLIGHT_CONF"
  195. fi
  196. }
  197. # stop_searchlight - Stop running processes
  198. function stop_searchlight {
  199. # Kill the searchlight screen windows
  200. stop_process searchlight-api
  201. stop_process searchlight-listener
  202. remove_uwsgi_config "$SEARCHLIGHT_UWSGI_CONF" "$SEARCHLIGHT_UWSGI_APP"
  203. }
  204. ###############
  205. # ELASTICSEARCH
  206. # Moving this here because the devstack team has determined that only
  207. # services supporting devstack core projects should live in devstack
  208. function _wget_elasticsearch {
  209. local baseurl=${1}
  210. local file=${2}
  211. if [ ! -f ${FILES}/${file} ]; then
  212. wget ${baseurl}/${file} -O ${FILES}/${file}
  213. fi
  214. if [ ! -f ${FILES}/${file}.sha1 ]; then
  215. # Starting with 2.0.0, sha1 files dropped the .txt extension and changed
  216. # the format slightly; need the leading spaces to comply with sha1sum
  217. ( wget ${baseurl}/${file}.sha1 -O ${FILES}/${file}.sha1 &&
  218. echo " ${file}" >> ${FILES}/${file}.sha1 )
  219. fi
  220. pushd ${FILES}; sha1sum ${file} > ${file}.sha1.gen; popd
  221. if ! diff ${FILES}/${file}.sha1.gen ${FILES}/${file}.sha1; then
  222. echo "Invalid elasticsearch download. Could not install."
  223. return 1
  224. else
  225. echo "SHA1 for ${file} matches downloaded ${FILES}/${file}.sha1"
  226. fi
  227. return 0
  228. }
  229. function _download_elasticsearch {
  230. if is_ubuntu; then
  231. arch="deb"
  232. elif is_fedora; then
  233. arch="rpm"
  234. else
  235. echo "Unknown architecture; can't download ElasticSearch"
  236. fi
  237. ELASTICSEARCH_FILENAME=elasticsearch-${ELASTICSEARCH_VERSION}.${arch}
  238. if [[ $ELASTICSEARCH_VERSION =~ ^2 ]]; then
  239. ELASTICSEARCH_URL=${ELASTICSEARCH_BASEURL_LEGACY}/${arch}/elasticsearch/${ELASTICSEARCH_VERSION}
  240. elif [[ $ELASTICSEARCH_VERSION =~ ^5 ]]; then
  241. ELASTICSEARCH_URL=${ELASTICSEARCH_BASEURL}
  242. else
  243. echo "Current Searchlight only supports ElasticSearch 2.x and 5.x"
  244. fi
  245. echo "Downloading ElasticSearch $ELASTICSEARCH_VERSION"
  246. echo "ElasticSearch URL is $ELASTICSEARCH_URL"
  247. _wget_elasticsearch $ELASTICSEARCH_URL $ELASTICSEARCH_FILENAME
  248. }
  249. function _check_elasticsearch_ready {
  250. # poll elasticsearch to see if it's started
  251. if ! wait_for_service 30 http://localhost:9200; then
  252. die $LINENO "Maximum timeout reached. Could not connect to ElasticSearch"
  253. fi
  254. }
  255. function _start_elasticsearch {
  256. echo "Starting elasticsearch"
  257. if is_ubuntu; then
  258. sudo /etc/init.d/elasticsearch start
  259. _check_elasticsearch_ready
  260. elif is_fedora; then
  261. sudo /bin/systemctl start elasticsearch.service
  262. _check_elasticsearch_ready
  263. else
  264. echo "Unsupported architecture... Can not start elasticsearch."
  265. fi
  266. }
  267. function _stop_elasticsearch {
  268. echo "Stopping elasticsearch"
  269. if is_ubuntu; then
  270. sudo /etc/init.d/elasticsearch stop
  271. elif is_fedora; then
  272. sudo /bin/systemctl stop elasticsearch.service
  273. else
  274. echo "Unsupported architecture...can not stop elasticsearch."
  275. fi
  276. }
  277. function _install_elasticsearch {
  278. # echo "Installing elasticsearch"
  279. # pip_install_gr elasticsearch
  280. if is_package_installed elasticsearch; then
  281. echo "Note: elasticsearch was already installed."
  282. return
  283. fi
  284. if is_ubuntu; then
  285. if [[ ${DISTRO} == "bionic" ]]; then
  286. is_package_installed openjdk-8-jdk-headless || install_package openjdk-8-jdk-headless
  287. else
  288. is_package_installed default-jdk-headless || install_package default-jdk-headless
  289. fi
  290. sudo dpkg -i ${FILES}/elasticsearch-${ELASTICSEARCH_VERSION}.deb
  291. sudo update-rc.d elasticsearch defaults 95 10
  292. elif is_fedora; then
  293. is_package_installed java-1.8.0-openjdk-headless || install_package java-1.8.0-openjdk-headless
  294. yum_install ${FILES}/elasticsearch-${ELASTICSEARCH_VERSION}.rpm
  295. sudo /bin/systemctl daemon-reload
  296. sudo /bin/systemctl enable elasticsearch.service
  297. else
  298. echo "Unsupported install of elasticsearch on this architecture."
  299. fi
  300. }
  301. function _uninstall_elasticsearch {
  302. echo "Uninstalling elasticsearch"
  303. if is_package_installed elasticsearch; then
  304. if is_ubuntu; then
  305. sudo apt-get purge elasticsearch
  306. elif is_fedora; then
  307. sudo yum remove elasticsearch
  308. else
  309. echo "Unsupported install of elasticsearch on this architecture."
  310. fi
  311. fi
  312. }
  313. #
  314. # END OF ELASTICSEARCH
  315. ######################
  316. # check for service enabled
  317. if is_service_enabled searchlight; then
  318. if [[ "$1" == "source" ]]; then
  319. # Initial source of lib script
  320. source $TOP_DIR/lib/searchlight
  321. fi
  322. if [[ "$1" == "stack" && "$2" == "install" ]]; then
  323. echo_summary "Installing Searchlight"
  324. install_searchlight
  325. echo_summary "Installing Searchlight client"
  326. install_searchlightclient
  327. elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
  328. echo_summary "Configuring Searchlight"
  329. configure_searchlight
  330. if is_service_enabled key; then
  331. echo_summary "Creating Searchlight Keystone Accounts"
  332. create_searchlight_accounts
  333. fi
  334. elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
  335. echo_summary "Initializing Searchlight"
  336. init_searchlight
  337. echo_summary "Starting Searchlight"
  338. start_searchlight
  339. fi
  340. if [[ "$1" == "unstack" ]]; then
  341. stop_searchlight
  342. fi
  343. if [[ "$1" == "clean" ]]; then
  344. echo_summary "Cleaning Searchlight"
  345. cleanup_searchlight
  346. fi
  347. fi
  348. # Restore xtrace
  349. $XTRACE