From 368466730c0637cce9ab99c54548d12be2c98c43 Mon Sep 17 00:00:00 2001
From: Ian Wienand <iwienand@redhat.com>
Date: Tue, 17 Nov 2020 17:13:46 +1100
Subject: [PATCH] Migrate codesearch site to container
The hound project has undergone a small re-birth and moved to
https://github.com/hound-search/hound
which has broken our deployment. We've talked about leaving
codesearch up to gitea, but it's not quite there yet. There seems to
be no point working on the puppet now.
This builds a container than runs houndd. It's an opendev specific
container; the config is pulled from project-config directly.
There's some custom scripts that drive things. Some points for
reviewers:
- update-hound-config.sh uses "create-hound-config" (which is in
jeepyb for historical reasons) to generate the config file. It
grabs the latest projects.yaml from project-config and exits with a
return code to indicate if things changed.
- when the container starts, it runs update-hound-config.sh to
populate the initial config. There is a testing environment flag
and small config so it doesn't have to clone the entire opendev for
functional testing.
- it runs under supervisord so we can restart the daemon when
projects are updated. Unlike earlier versions that didn't start
listening till indexing was done, this version now puts up a "Hound
is not ready yet" message when while it is working; so we can drop
all the magic we were doing to probe if hound is listening via
netstat and making Apache redirect to a status page.
- resync-hound.sh is run from an external cron job daily, and does
this update and restart check. Since it only reloads if changes
are made, this should be relatively rare anyway.
- There is a PR to monitor the config file
(https://github.com/hound-search/hound/pull/357) which would mean
the restart is unnecessary. This would be good in the near and we
could remove the cron job.
- playbooks/roles/codesearch is unexciting and deploys the container,
certificates and an apache proxy back to localhost:6080 where hound
is listening.
I've combined removal of the old puppet bits here as the "-codesearch"
namespace was already being used.
Change-Id: I8c773b5ea6b87e8f7dfd8db2556626f7b2500473
---
doc/source/codesearch.rst | 18 ++---
docker/hound/Dockerfile | 37 +++++++++
docker/hound/resync-hound.sh | 16 ++++
docker/hound/sample-projects.yaml | 15 ++++
docker/hound/start-container.sh | 9 +++
docker/hound/supervisord.conf | 19 +++++
docker/hound/update-hound-config.sh | 34 ++++++++
inventory/base/hosts.yaml | 7 --
inventory/service/groups.yaml | 7 +-
.../host_vars/codesearch01.opendev.org.yaml | 5 ++
manifests/codesearch.pp | 5 --
.../files/resync-hound-config.sh | 64 ---------------
.../openstack_project/manifests/codesearch.pp | 54 -------------
playbooks/roles/codesearch/README.rst | 1 +
playbooks/roles/codesearch/defaults/main.yaml | 1 +
playbooks/roles/codesearch/handlers/main.yaml | 4 +
playbooks/roles/codesearch/tasks/main.yaml | 78 +++++++++++++++++++
.../codesearch/templates/codesearch.vhost.j2 | 41 ++++++++++
.../templates/docker-compose.yaml.j2 | 15 ++++
.../handlers/main.yaml | 3 +
playbooks/service-codesearch.yaml | 17 +---
playbooks/zuul/run-base.yaml | 1 +
.../codesearch01.opendev.org.yaml.j2 | 1 +
testinfra/test_codesearch.py | 27 +++++++
zuul.d/docker-images/hound.yaml | 27 +++++++
zuul.d/infra-prod.yaml | 43 +++++-----
zuul.d/project.yaml | 25 +++++-
zuul.d/system-config-run.yaml | 60 +++++++-------
28 files changed, 419 insertions(+), 215 deletions(-)
create mode 100644 docker/hound/Dockerfile
create mode 100755 docker/hound/resync-hound.sh
create mode 100644 docker/hound/sample-projects.yaml
create mode 100755 docker/hound/start-container.sh
create mode 100644 docker/hound/supervisord.conf
create mode 100755 docker/hound/update-hound-config.sh
create mode 100644 inventory/service/host_vars/codesearch01.opendev.org.yaml
delete mode 100644 manifests/codesearch.pp
delete mode 100644 modules/openstack_project/files/resync-hound-config.sh
delete mode 100644 modules/openstack_project/manifests/codesearch.pp
create mode 100644 playbooks/roles/codesearch/README.rst
create mode 100644 playbooks/roles/codesearch/defaults/main.yaml
create mode 100644 playbooks/roles/codesearch/handlers/main.yaml
create mode 100644 playbooks/roles/codesearch/tasks/main.yaml
create mode 100644 playbooks/roles/codesearch/templates/codesearch.vhost.j2
create mode 100644 playbooks/roles/codesearch/templates/docker-compose.yaml.j2
create mode 100644 playbooks/zuul/templates/host_vars/codesearch01.opendev.org.yaml.j2
create mode 100644 testinfra/test_codesearch.py
create mode 100644 zuul.d/docker-images/hound.yaml
diff --git a/doc/source/codesearch.rst b/doc/source/codesearch.rst
index 34eaa76167..9e3f315033 100644
--- a/doc/source/codesearch.rst
+++ b/doc/source/codesearch.rst
@@ -5,31 +5,29 @@
Code Search
###########
-The `Hound <https://github.com/etsy/Hound>`_ code search engine is deployed in
-our infrastructure to service all OpenStack repositories.
+The `Hound <https://github.com/hound-search/hound>`_ code search
+engine is deployed in our infrastructure to service all OpenStack
+repositories.
At a Glance
===========
:Hosts:
- * http://codesearch.openstack.org
+ * http://codesearch.opendev.org
:Puppet:
- * https://opendev.org/opendev/puppet-hound
- * :git_file:`modules/openstack_project/manifests/codesearch.pp`
+ * :git_file:`playbooks/roles/codesearch`
:Projects:
- * https://github.com/etsy/Hound
+ * https://github.com/hound-search/hound
:Bugs:
* https://storyboard.openstack.org/#!/project/748
- * https://github.com/etsy/Hound/issues
:Resources:
- * `Hound README <https://github.com/etsy/hound/blob/master/README.md>`_
+ * `Hound README <https://github.com/hound-search/hound/blob/master/README.md>`_
Overview
========
Hound is configured to read projects from a config.json file that is
-automatically generated from the Gerrit projects.yaml, defined in the
-$::project_config::jeepyb_project_file variable in Puppet.
+automatically generated from the Gerrit projects.yaml
Maintenance
diff --git a/docker/hound/Dockerfile b/docker/hound/Dockerfile
new file mode 100644
index 0000000000..76c2fb48a7
--- /dev/null
+++ b/docker/hound/Dockerfile
@@ -0,0 +1,37 @@
+# Copyright (c) 2020 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+FROM docker.io/opendevorg/python-base:3.8
+
+ENV GOPATH /go
+
+RUN apt-get update \
+ && apt-get install -y curl golang git
+
+RUN go get github.com/hound-search/hound/cmds/...
+
+RUN pip install git+https://opendev.org/opendev/jeepyb#egg=jeepyb \
+ supervisor
+
+RUN apt-get clean \
+ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+ADD supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+ADD start-container.sh /usr/bin/start-container
+ADD update-hound-config.sh /usr/local/bin/update-hound-config
+ADD resync-hound.sh /usr/local/bin/resync-hound
+ADD sample-projects.yaml /var/run/sample-projects.yaml
+
+ENTRYPOINT ["start-container"]
+
diff --git a/docker/hound/resync-hound.sh b/docker/hound/resync-hound.sh
new file mode 100755
index 0000000000..c209d39208
--- /dev/null
+++ b/docker/hound/resync-hound.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+rc=0
+
+update-hound-config || rc=$?
+
+if [[ ${rc} == 2 ]]; then
+ echo "No project modified"
+ exit 0
+elif [[ ${rc} == 0 ]]; then
+ echo "*** New projects found, restarting houndd"
+ supervisorctl restart houndd
+else
+ echo "*** Unknown exit: ${rc}"
+ exit ${rc}
+fi
diff --git a/docker/hound/sample-projects.yaml b/docker/hound/sample-projects.yaml
new file mode 100644
index 0000000000..ff3da1aaed
--- /dev/null
+++ b/docker/hound/sample-projects.yaml
@@ -0,0 +1,15 @@
+- project: opendev/system-config
+ use-storyboard: true
+ groups:
+ - openstack-ci
+ description: System configuration for OpenStack Infrastructure
+- project: openstack/project-config
+ use-storyboard: true
+ groups:
+ - openstack-ci
+ description: Configuration files for project CI systems
+- project: zuul/zuul
+ use-storyboard: true
+ groups:
+ - zuul
+ description: The Gatekeeper, or a project gating system
diff --git a/docker/hound/start-container.sh b/docker/hound/start-container.sh
new file mode 100755
index 0000000000..33be187aa9
--- /dev/null
+++ b/docker/hound/start-container.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+if [ $# -gt 0 ]; then
+ exec "$@"
+else
+ if [ ! -f /var/run/config.json ]; then
+ update-hound-config;
+ fi
+ /usr/local/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
+fi
diff --git a/docker/hound/supervisord.conf b/docker/hound/supervisord.conf
new file mode 100644
index 0000000000..d19316792e
--- /dev/null
+++ b/docker/hound/supervisord.conf
@@ -0,0 +1,19 @@
+[supervisord]
+nodaemon = true
+
+[supervisorctl]
+
+[inet_http_server]
+port = 127.0.0.1:9001
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[program:houndd]
+directory=/var/run
+command=/go/bin/houndd -conf /var/run/config.json
+logfile_maxbytes=0
+stdout_logfile_maxbytes=0
+stderr_logfile_maxbytes=0
+stdout_logfile=/dev/stdout
+stderr_logfile=/dev/stdout
diff --git a/docker/hound/update-hound-config.sh b/docker/hound/update-hound-config.sh
new file mode 100755
index 0000000000..912322c05a
--- /dev/null
+++ b/docker/hound/update-hound-config.sh
@@ -0,0 +1,34 @@
+#!/bin/bash -x
+
+CONFIG_DIR=/var/run
+PROJECTS_FILE_NAME=projects.yaml
+CONFIG_FILE_NAME=config.json
+PROJECTS_FILE=${CONFIG_DIR}/${PROJECTS_FILE_NAME}
+CONFIG_FILE=${CONFIG_DIR}/${CONFIG_FILE_NAME}
+
+PROJECT_CONFIG=https://opendev.org/openstack/project-config/raw/branch/master/gerrit/projects.yaml
+
+pushd $CONFIG_DIR
+
+# 2 signals nothing done, 0 means updated
+_exit=2
+
+if [ ${USE_HOUND_TEST_CONFIG:-} = 1 ]; then
+ PROJECTS_YAML=/var/run/sample-projects.yaml create-hound-config
+ exit 0
+fi
+
+curl -o ${PROJECTS_FILE}.tmp ${PROJECT_CONFIG}
+md5sum ${PROJECTS_FILE}.tmp > ${PROJECTS_FILE}.tmp.md5
+
+if [ ! -f ${PROJECTS_FILE} ] || \
+ ! cmp --silent ${PROJECTS_FILE}.md5 ${PROJECTS_FILE}.tmp.md5; then
+ mv ${PROJECTS_FILE}.tmp ${PROJECTS_FILE}
+ mv ${PROJECTS_FILE}.tmp.md5 ${PROJECTS_FILE}.md5
+ PROJECTS_YAML=${PROJECTS_FILE} create-hound-config
+ _exit=0
+fi
+
+popd
+
+exit $_exit
diff --git a/inventory/base/hosts.yaml b/inventory/base/hosts.yaml
index d17c1b2949..c1fe0f00ca 100644
--- a/inventory/base/hosts.yaml
+++ b/inventory/base/hosts.yaml
@@ -91,13 +91,6 @@ all:
region_name: DFW
public_v4: 172.99.116.215
public_v6: 2001:4800:7821:105:be76:4eff:fe04:b9a5
- codesearch01.openstack.org:
- ansible_host: 23.253.92.77
- location:
- cloud: openstackci-rax
- region_name: DFW
- public_v4: 23.253.92.77
- public_v6: 2001:4800:7815:105:be76:4eff:fe04:5fdf
eavesdrop01.openstack.org:
ansible_host: 104.130.124.113
location:
diff --git a/inventory/service/groups.yaml b/inventory/service/groups.yaml
index 0341c936c3..83320faecc 100644
--- a/inventory/service/groups.yaml
+++ b/inventory/service/groups.yaml
@@ -56,7 +56,7 @@ groups:
cloud-launcher:
- bridge.openstack.org
codesearch:
- - codesearch[0-9]*.open*.org
+ - codesearch[0-9]*.opendev.org
control-plane-clouds:
- bridge.openstack.org
disabled:
@@ -93,6 +93,7 @@ groups:
kdc:
- kdc[0-9]*.open*.org
letsencrypt:
+ - codesearch[0-9]*.opendev.org
- etherpad[0-9]*.opendev.org
- gitea[0-9]*.opendev.org
- graphite[0-9]*.opendev.org
@@ -143,7 +144,6 @@ groups:
- ask*.open*.org
- backup[0-9]*.openstack.org
- cacti[0-9]*.open*.org
- - codesearch[0-9]*.open*.org
- corvustest
- eavesdrop[0-9]*.open*.org
- elasticsearch[0-9]*.open*.org
@@ -178,7 +178,6 @@ groups:
- ask*.open*.org
- ask-staging[0-9]*.open*.org
- cacti[0-9]*.open*.org
- - codesearch[0-9]*.open*.org
- eavesdrop[0-9]*.open*.org
- elasticsearch[0-9]*.open*.org
- ethercalc[0-9]*.open*.org
@@ -234,7 +233,7 @@ groups:
webservers:
- ask*.open*.org
- cacti[0-9]*.open*.org
- - codesearch[0-9]*.open*.org
+ - codesearch[0-9]*.opendev.org
- eavesdrop[0-9]*.open*.org
- ethercalc[0-9]*.open*.org
- etherpad[0-9]*.open*.org
diff --git a/inventory/service/host_vars/codesearch01.opendev.org.yaml b/inventory/service/host_vars/codesearch01.opendev.org.yaml
new file mode 100644
index 0000000000..3492f653cd
--- /dev/null
+++ b/inventory/service/host_vars/codesearch01.opendev.org.yaml
@@ -0,0 +1,5 @@
+letsencrypt_certs:
+ codesearch01-opendev-org-main:
+ - codesearch01.opendev.org
+ - codesearch.opendev.org
+ - codesearch.openstack.org
diff --git a/manifests/codesearch.pp b/manifests/codesearch.pp
deleted file mode 100644
index 41f45cdb9a..0000000000
--- a/manifests/codesearch.pp
+++ /dev/null
@@ -1,5 +0,0 @@
-# Node-OS: xenial
-node /^codesearch\d*\.open.*\.org$/ {
- $group = "codesearch"
- class { 'openstack_project::codesearch': }
-}
diff --git a/modules/openstack_project/files/resync-hound-config.sh b/modules/openstack_project/files/resync-hound-config.sh
deleted file mode 100644
index ec6c217682..0000000000
--- a/modules/openstack_project/files/resync-hound-config.sh
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/bin/bash
-
-# Copyright 2017 Red Hat, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-PROJECTS_YAML=${PROJECTS_YAML:-/etc/project-config/gerrit/projects.yaml}
-REINDEX_LOCK=/var/www/hound/reindex.lock
-
-TEMP_DIR=$(mktemp -d)
-trap "rm -rf ${TEMP_DIR} EXIT"
-
-pushd ${TEMP_DIR}
-
-echo $(date)
-echo "Starting hound config update"
-
-# Generate the new config
-PROJECTS_YAML=${PROJECTS_YAML} create-hound-config
-
-# See if we need to update
-NEW="$(md5sum config.json | awk '{print $1}')"
-OLD="$(md5sum /home/hound/config.json | awk '{print $1}')"
-if [[ ${NEW} == ${OLD} ]]; then
- echo "Nothing to do"
- exit 0
-fi
-
-echo "Recreating config"
-
-# Move the new config into place
-chown hound:hound config.json
-chmod 0644 config.json
-cp /home/hound/config.json /home/hound/config.json.bak
-mv ./config.json /home/hound/config.json
-
-# release the hounds
-touch ${REINDEX_LOCK}
-service hound stop
-sleep 2
-service hound start
-
-# Hound takes a few minutes to go through all our projects. We know
-# it's ready when we see it listening on port 6080
-echo "Waiting for hound..."
-while ! netstat -lnt | grep -q ':6080.*LISTEN\s*$' ; do
- echo " ... still waiting"
- sleep 5
-done
-
-rm ${REINDEX_LOCK}
-
-echo "... done"
-
diff --git a/modules/openstack_project/manifests/codesearch.pp b/modules/openstack_project/manifests/codesearch.pp
deleted file mode 100644
index 8a5212e24b..0000000000
--- a/modules/openstack_project/manifests/codesearch.pp
+++ /dev/null
@@ -1,54 +0,0 @@
-# Class to configure hound on a node.
-class openstack_project::codesearch {
-
- class { 'hound':
- manage_config => false,
- }
-
- include ::jeepyb
- include ::logrotate
- include ::pip
-
- file { '/home/hound/config.json':
- ensure => 'present',
- }
-
- file { '/usr/local/bin/resync-hound-config':
- ensure => present,
- owner => 'root',
- group => 'root',
- mode => '0755',
- source => 'puppet:///modules/openstack_project/resync-hound-config.sh',
- }
-
- # Note: we could trigger this from project-config changes, but it
- # does bring the service down for several minutes if something
- # changes. Once a day should be enough.
- cron { 'hound':
- user => root,
- hour => '4',
- minute => '0',
- command => 'flock -n /var/run/hound.sync.lock resync-hound-config >> /var/log/hound.sync.log 2>&1',
- environment => [
- 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
- "PROJECTS_YAML=/opt/project-config/projects.yaml",
- ],
- require => [
- File['/usr/local/bin/resync-hound-config'],
- File['/home/hound/config.json'],
- ],
- }
-
- logrotate::file { 'hound-sync':
- log => '/var/log/hound.sync.log',
- options => [
- 'compress',
- 'copytruncate',
- 'missingok',
- 'rotate 7',
- 'daily',
- 'notifempty',
- ],
- }
-
-}
diff --git a/playbooks/roles/codesearch/README.rst b/playbooks/roles/codesearch/README.rst
new file mode 100644
index 0000000000..cac72a9156
--- /dev/null
+++ b/playbooks/roles/codesearch/README.rst
@@ -0,0 +1 @@
+Run a hound container to index Opendev code
diff --git a/playbooks/roles/codesearch/defaults/main.yaml b/playbooks/roles/codesearch/defaults/main.yaml
new file mode 100644
index 0000000000..55bb2b2c54
--- /dev/null
+++ b/playbooks/roles/codesearch/defaults/main.yaml
@@ -0,0 +1 @@
+codesearch_use_test_config: False
diff --git a/playbooks/roles/codesearch/handlers/main.yaml b/playbooks/roles/codesearch/handlers/main.yaml
new file mode 100644
index 0000000000..d24ae5182f
--- /dev/null
+++ b/playbooks/roles/codesearch/handlers/main.yaml
@@ -0,0 +1,4 @@
+- name: codesearch Reload apache2
+ service:
+ name: apache2
+ state: reloaded
diff --git a/playbooks/roles/codesearch/tasks/main.yaml b/playbooks/roles/codesearch/tasks/main.yaml
new file mode 100644
index 0000000000..b9a71ac025
--- /dev/null
+++ b/playbooks/roles/codesearch/tasks/main.yaml
@@ -0,0 +1,78 @@
+- name: Ensure docker-compose directory exists
+ file:
+ state: directory
+ path: /etc/hound-docker
+
+- name: Write settings file
+ template:
+ src: docker-compose.yaml.j2
+ dest: /etc/hound-docker/docker-compose.yaml
+
+- name: Install apache2
+ apt:
+ name:
+ - apache2
+ - apache2-utils
+ state: present
+
+- name: Apache modules
+ apache2_module:
+ state: present
+ name: "{{ item }}"
+ loop:
+ - rewrite
+ - proxy
+ - proxy_http
+ - ssl
+ - headers
+ - proxy_wstunnel
+
+- name: Copy apache config
+ template:
+ src: codesearch.vhost.j2
+ dest: /etc/apache2/sites-enabled/000-default.conf
+ owner: root
+ group: root
+ mode: 0644
+ notify: codesearch Reload apache2
+
+- name: Create hound data storage area
+ file:
+ state: directory
+ path: /var/lib/hound/data
+ owner: root
+ group: root
+ mode: 0755
+
+- name: Run docker-compose pull
+ shell:
+ cmd: docker-compose pull
+ chdir: /etc/hound-docker/
+
+- name: Run docker-compose up
+ shell:
+ cmd: docker-compose up -d
+ chdir: /etc/hound-docker/
+
+- name: Run docker prune to cleanup unneeded images
+ shell:
+ cmd: docker image prune -f
+
+# Daily update of codesearch. This only reloads hound
+# if the project-config yaml has changed
+- name: Install update cron job
+ cron:
+ name: Update codesearch
+ state: present
+ user: root
+ job: >
+ /usr/local/bin/docker-compose -f /etc/hound-docker/docker-compose.yaml exec -T hound
+ /usr/local/bin/resync-hound >> /var/log/resync-hound.log 2>&1
+ hour: 5
+ minute: 30
+
+- name: Rotate sync logs
+ include_role:
+ name: logrotate
+ vars:
+ logrotate_file_name: /var/log/resync-hound.log
diff --git a/playbooks/roles/codesearch/templates/codesearch.vhost.j2 b/playbooks/roles/codesearch/templates/codesearch.vhost.j2
new file mode 100644
index 0000000000..17549fa59a
--- /dev/null
+++ b/playbooks/roles/codesearch/templates/codesearch.vhost.j2
@@ -0,0 +1,41 @@
+<VirtualHost *:80>
+ ServerName {{ inventory_hostname }}
+ ServerAdmin infra-root@openstack.org
+
+ ErrorLog ${APACHE_LOG_DIR}/codesearch-error.log
+
+ LogLevel warn
+
+ CustomLog ${APACHE_LOG_DIR}/codesearch-access.log combined
+
+ Redirect / https://codesearch.opendev.org/
+
+</VirtualHost>
+
+<VirtualHost *:443>
+ ServerName {{ inventory_hostname }}
+ ServerAdmin webmaster@openstack.org
+
+ AllowEncodedSlashes On
+
+ ErrorLog ${APACHE_LOG_DIR}/codesearch-ssl-error.log
+
+ LogLevel warn
+
+ CustomLog ${APACHE_LOG_DIR}/codesearch-ssl-access.log combined
+
+ SSLEngine on
+ SSLProtocol All -SSLv2 -SSLv3
+ # Note: this list should ensure ciphers that provide forward secrecy
+ SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
+ SSLHonorCipherOrder on
+
+ SSLCertificateFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.cer
+ SSLCertificateKeyFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.key
+ SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer
+
+ ProxyPass / http://localhost:6080/ retry=0
+ ProxyPassReverse / http://localhost:6080/
+
+</VirtualHost>
+
diff --git a/playbooks/roles/codesearch/templates/docker-compose.yaml.j2 b/playbooks/roles/codesearch/templates/docker-compose.yaml.j2
new file mode 100644
index 0000000000..460eb83499
--- /dev/null
+++ b/playbooks/roles/codesearch/templates/docker-compose.yaml.j2
@@ -0,0 +1,15 @@
+version: '3'
+
+services:
+ hound:
+ restart: always
+ image: docker.io/opendevorg/hound
+ network_mode: host
+ environment:
+ - 'USE_HOUND_TEST_CONFIG={{ "1" if codesearch_use_test_config else "0" }}'
+ volumes:
+ - /var/lib/hound/data:/var/run/data
+ logging:
+ driver: syslog
+ options:
+ tag: "docker-hound"
diff --git a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml
index 0d8a48c9f5..be04e2f520 100644
--- a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml
+++ b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml
@@ -120,6 +120,9 @@
- name: letsencrypt updated grafana01-opendev-org-main
include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+- name: letsencrypt updated codesearch01-opendev-org-main
+ include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+
# nodepool
- name: letsencrypt updated nb01-opendev-org-main
diff --git a/playbooks/service-codesearch.yaml b/playbooks/service-codesearch.yaml
index fb6e7d9167..9a86ef49b5 100644
--- a/playbooks/service-codesearch.yaml
+++ b/playbooks/service-codesearch.yaml
@@ -1,15 +1,6 @@
-- hosts: 'localhost:!disabled'
- name: Install puppet role/modules
- strategy: linear
- roles:
- - puppet-setup-ansible
-
-- hosts: 'codesearch:!disabled'
- name: "codesearch: run puppet on codesearch"
- strategy: free
+- hosts: "codesearch:!disabled"
+ name: "Configure codesearch"
roles:
- iptables
- - sync-project-config
- - pip3
- - name: puppet-run
- manifest: /opt/system-config/production/manifests/codesearch.pp
+ - install-docker
+ - codesearch
diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml
index 44e7f51cd6..42d7bd1d16 100644
--- a/playbooks/zuul/run-base.yaml
+++ b/playbooks/zuul/run-base.yaml
@@ -69,6 +69,7 @@
- group_vars/zuul-scheduler.yaml
- group_vars/zuul-web.yaml
- host_vars/bridge.openstack.org.yaml
+ - host_vars/codesearch01.opendev.org.yaml
- host_vars/etherpad01.opendev.org.yaml
- host_vars/letsencrypt01.opendev.org.yaml
- host_vars/letsencrypt02.opendev.org.yaml
diff --git a/playbooks/zuul/templates/host_vars/codesearch01.opendev.org.yaml.j2 b/playbooks/zuul/templates/host_vars/codesearch01.opendev.org.yaml.j2
new file mode 100644
index 0000000000..d0186e9179
--- /dev/null
+++ b/playbooks/zuul/templates/host_vars/codesearch01.opendev.org.yaml.j2
@@ -0,0 +1 @@
+codesearch_use_test_config: True
diff --git a/testinfra/test_codesearch.py b/testinfra/test_codesearch.py
new file mode 100644
index 0000000000..a057caf7cd
--- /dev/null
+++ b/testinfra/test_codesearch.py
@@ -0,0 +1,27 @@
+# Copyright 2020 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+testinfra_hosts = ['codesearch01.opendev.org']
+
+
+def test_codesearch_container_listening(host):
+ codesearch = host.socket("tcp://127.0.0.1:6080")
+ assert codesearch.is_listening
+
+def test_codesearch_proxy(host):
+ cmd = host.run('curl --insecure '
+ '--resolve codesearch.opendev.org:443:127.0.0.1 '
+ 'https://codesearch.opendev.org')
+ assert '<title>Hound</title>' in cmd.stdout
diff --git a/zuul.d/docker-images/hound.yaml b/zuul.d/docker-images/hound.yaml
new file mode 100644
index 0000000000..aa6d390d91
--- /dev/null
+++ b/zuul.d/docker-images/hound.yaml
@@ -0,0 +1,27 @@
+# Hound jobs
+- job:
+ name: system-config-build-image-hound
+ description: Build a hound image.
+ provides: hound-container-image
+ parent: system-config-build-image
+ vars: &hound_vars
+ docker_images:
+ - context: docker/hound
+ repository: opendevorg/hound
+ files: &hound_files
+ - docker/hound/
+
+- job:
+ name: system-config-upload-image-hound
+ description: Build and upload a hound image.
+ provides: hound-container-image
+ parent: system-config-upload-image
+ vars: *hound_vars
+ files: *hound_files
+
+- job:
+ name: system-config-promote-image-hound
+ description: Promote a previously published hound image to latest.
+ parent: system-config-promote-image
+ vars: *hound_vars
+ files: *hound_files
diff --git a/zuul.d/infra-prod.yaml b/zuul.d/infra-prod.yaml
index 72602d68e6..16070978ac 100644
--- a/zuul.d/infra-prod.yaml
+++ b/zuul.d/infra-prod.yaml
@@ -448,31 +448,6 @@
- docker/jinja-init/
- docker/python-base/
-- job:
- name: infra-prod-service-codesearch
- parent: infra-prod-service-base
- description: Run service-codesearch.yaml playbook.
- required-projects:
- - opendev/ansible-role-puppet
- - opendev/system-config
- - openstack/project-config
- vars:
- playbook_name: service-codesearch.yaml
- files:
- - inventory/
- - playbooks/install-ansible.yaml
- - playbooks/service-codesearch.yaml
- - inventory/service/group_vars/puppet.yaml
- - playbooks/roles/run-puppet/
- - playbooks/roles/install-ansible-roles/
- - playbooks/roles/iptables/
- - playbooks/roles/sync-project-config
- - playbooks/roles/puppet-install/
- - playbooks/roles/disable-puppet-agent/
- - modules/openstack_project/manifests/codesearch.pp
- - modules/openstack_project/files/resync-hound-config.sh
- - manifests/codesearch.pp
-
- job:
name: infra-prod-service-eavesdrop
parent: infra-prod-service-base
@@ -526,6 +501,24 @@
- playbooks/roles/accessbot
- docker/accessbot/
+- job:
+ name: infra-prod-service-codesearch
+ parent: infra-prod-service-base
+ description: Run service-codesearch.yaml playbook.
+ vars:
+ playbook_name: service-codesearch.yaml
+ files:
+ - docker/hound/
+ - inventory/
+ - playbooks/service-codesearch.yaml
+ - inventory/service/host_vars/codesearch01.opendev.yaml
+ - inventory/service/group_vars/codesearch
+ - playbooks/roles/install-docker/
+ - playbooks/roles/pip3/
+ - playbooks/roles/codesearch
+ - playbooks/roles/logrotate
+ - playbooks/roles/iptables
+
- job:
name: infra-prod-service-grafana
parent: infra-prod-service-base
diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
index 2bf2cccfd5..449d0aaa35 100644
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@@ -21,7 +21,11 @@
- name: opendev-buildset-registry
- name: system-config-build-image-accessbot
soft: true
- - system-config-run-codesearch
+ - system-config-run-codesearch:
+ dependencies:
+ - name: opendev-buildset-registry
+ - name: system-config-build-image-hound
+ soft: true
- system-config-run-lists
- system-config-run-nodepool
- system-config-run-meetpad:
@@ -70,6 +74,11 @@
- name: opendev-buildset-registry
- name: system-config-build-image-jinja-init
soft: true
+ - system-config-build-image-hound:
+ dependencies:
+ - name: opendev-buildset-registry
+ - name: system-config-build-image-python-base-3.8
+ soft: true
- system-config-build-image-etherpad
- system-config-build-image-gitea
- system-config-build-image-grafana
@@ -107,7 +116,11 @@
- name: opendev-buildset-registry
- name: system-config-upload-image-accessbot
soft: true
- - system-config-run-codesearch
+ - system-config-run-codesearch:
+ dependencies:
+ - name: opendev-buildset-registry
+ - name: system-config-upload-image-hound
+ soft: true
- system-config-run-lists
- system-config-run-nodepool
- system-config-run-meetpad:
@@ -156,6 +169,7 @@
- name: opendev-buildset-registry
- name: system-config-upload-image-jinja-init
soft: true
+ - system-config-upload-image-hound
- system-config-upload-image-etherpad
- system-config-upload-image-gitea
- system-config-upload-image-grafana
@@ -181,6 +195,7 @@
- opendev-promote-docs
deploy:
jobs:
+ - system-config-promote-image-hound
- system-config-promote-image-jinja-init
- system-config-promote-image-gitea-init
- system-config-promote-image-gitea
@@ -218,6 +233,12 @@
- infra-prod-service-gitea-lb
- infra-prod-service-nameserver
- infra-prod-service-nodepool
+ - infra-prod-service-codesearch:
+ dependencies:
+ - name: infra-prod-letsencrypt
+ soft: true
+ - name: system-config-promote-image-hound
+ soft: true
- infra-prod-service-etherpad:
dependencies:
- name: infra-prod-install-ansible
diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml
index e9a9fc9ff8..a2c64ddd1d 100644
--- a/zuul.d/system-config-run.yaml
+++ b/zuul.d/system-config-run.yaml
@@ -156,37 +156,6 @@
- docker/accessbot/
- testinfra/test_eavesdrop.py
-- job:
- name: system-config-run-codesearch
- parent: system-config-run
- description: |
- Run the playbook for an codesearch server.
- nodeset:
- nodes:
- - name: bridge.openstack.org
- label: ubuntu-bionic
- - name: codesearch01.openstack.org
- label: ubuntu-xenial
- required-projects:
- - opendev/ansible-role-puppet
- - opendev/system-config
- - openstack/project-config
- files:
- - playbooks/install-ansible.yaml
- - playbooks/service-codesearch.yaml
- - inventory/service/group_vars/puppet.yaml
- - playbooks/roles/run-puppet/
- - playbooks/roles/install-ansible-roles/
- - playbooks/roles/sync-project-config
- - playbooks/roles/puppet-install/
- - playbooks/roles/disable-puppet-agent/
- - modules/openstack_project/manifests/codesearch.pp
- - modules/openstack_project/files/resync-hound-config.sh
- - manifests/codesearch.pp
- vars:
- run_playbooks:
- - playbooks/service-codesearch.yaml
-
- job:
name: system-config-run-letsencrypt
parent: system-config-run
@@ -501,6 +470,35 @@
- playbooks/roles/install-docker/
- testinfra/test_registry.py
+- job:
+ name: system-config-run-codesearch
+ parent: system-config-run-containers
+ description: |
+ Run the playbook for the codesearch server.
+ timeout: 3600
+ requires: codesearch-container-image
+ required-projects:
+ - opendev/system-config
+ nodeset:
+ nodes:
+ - name: bridge.openstack.org
+ label: ubuntu-bionic
+ - name: codesearch01.opendev.org
+ label: ubuntu-focal
+ vars:
+ run_playbooks:
+ - playbooks/letsencrypt.yaml
+ - playbooks/service-codesearch.yaml
+ files:
+ - playbooks/bridge.yaml
+ - playbooks/letsencrypt.yaml
+ - playbooks/service-codesearch.yaml
+ - playbooks/roles/codesearch/
+ - playbooks/roles/install-docker/
+ - playbooks/roles/pip3/
+ - docker/codesearch/
+ - testinfra/test_codesearch.py
+
- job:
name: system-config-run-etherpad
parent: system-config-run-containers