From 368466730c0637cce9ab99c54548d12be2c98c43 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Tue, 17 Nov 2020 17:13:46 +1100 Subject: [PATCH] Migrate codesearch site to container The hound project has undergone a small re-birth and moved to https://github.com/hound-search/hound which has broken our deployment. We've talked about leaving codesearch up to gitea, but it's not quite there yet. There seems to be no point working on the puppet now. This builds a container than runs houndd. It's an opendev specific container; the config is pulled from project-config directly. There's some custom scripts that drive things. Some points for reviewers: - update-hound-config.sh uses "create-hound-config" (which is in jeepyb for historical reasons) to generate the config file. It grabs the latest projects.yaml from project-config and exits with a return code to indicate if things changed. - when the container starts, it runs update-hound-config.sh to populate the initial config. There is a testing environment flag and small config so it doesn't have to clone the entire opendev for functional testing. - it runs under supervisord so we can restart the daemon when projects are updated. Unlike earlier versions that didn't start listening till indexing was done, this version now puts up a "Hound is not ready yet" message when while it is working; so we can drop all the magic we were doing to probe if hound is listening via netstat and making Apache redirect to a status page. - resync-hound.sh is run from an external cron job daily, and does this update and restart check. Since it only reloads if changes are made, this should be relatively rare anyway. - There is a PR to monitor the config file (https://github.com/hound-search/hound/pull/357) which would mean the restart is unnecessary. This would be good in the near and we could remove the cron job. - playbooks/roles/codesearch is unexciting and deploys the container, certificates and an apache proxy back to localhost:6080 where hound is listening. I've combined removal of the old puppet bits here as the "-codesearch" namespace was already being used. Change-Id: I8c773b5ea6b87e8f7dfd8db2556626f7b2500473 --- doc/source/codesearch.rst | 18 ++--- docker/hound/Dockerfile | 37 +++++++++ docker/hound/resync-hound.sh | 16 ++++ docker/hound/sample-projects.yaml | 15 ++++ docker/hound/start-container.sh | 9 +++ docker/hound/supervisord.conf | 19 +++++ docker/hound/update-hound-config.sh | 34 ++++++++ inventory/base/hosts.yaml | 7 -- inventory/service/groups.yaml | 7 +- .../host_vars/codesearch01.opendev.org.yaml | 5 ++ manifests/codesearch.pp | 5 -- .../files/resync-hound-config.sh | 64 --------------- .../openstack_project/manifests/codesearch.pp | 54 ------------- playbooks/roles/codesearch/README.rst | 1 + playbooks/roles/codesearch/defaults/main.yaml | 1 + playbooks/roles/codesearch/handlers/main.yaml | 4 + playbooks/roles/codesearch/tasks/main.yaml | 78 +++++++++++++++++++ .../codesearch/templates/codesearch.vhost.j2 | 41 ++++++++++ .../templates/docker-compose.yaml.j2 | 15 ++++ .../handlers/main.yaml | 3 + playbooks/service-codesearch.yaml | 17 +--- playbooks/zuul/run-base.yaml | 1 + .../codesearch01.opendev.org.yaml.j2 | 1 + testinfra/test_codesearch.py | 27 +++++++ zuul.d/docker-images/hound.yaml | 27 +++++++ zuul.d/infra-prod.yaml | 43 +++++----- zuul.d/project.yaml | 25 +++++- zuul.d/system-config-run.yaml | 60 +++++++------- 28 files changed, 419 insertions(+), 215 deletions(-) create mode 100644 docker/hound/Dockerfile create mode 100755 docker/hound/resync-hound.sh create mode 100644 docker/hound/sample-projects.yaml create mode 100755 docker/hound/start-container.sh create mode 100644 docker/hound/supervisord.conf create mode 100755 docker/hound/update-hound-config.sh create mode 100644 inventory/service/host_vars/codesearch01.opendev.org.yaml delete mode 100644 manifests/codesearch.pp delete mode 100644 modules/openstack_project/files/resync-hound-config.sh delete mode 100644 modules/openstack_project/manifests/codesearch.pp create mode 100644 playbooks/roles/codesearch/README.rst create mode 100644 playbooks/roles/codesearch/defaults/main.yaml create mode 100644 playbooks/roles/codesearch/handlers/main.yaml create mode 100644 playbooks/roles/codesearch/tasks/main.yaml create mode 100644 playbooks/roles/codesearch/templates/codesearch.vhost.j2 create mode 100644 playbooks/roles/codesearch/templates/docker-compose.yaml.j2 create mode 100644 playbooks/zuul/templates/host_vars/codesearch01.opendev.org.yaml.j2 create mode 100644 testinfra/test_codesearch.py create mode 100644 zuul.d/docker-images/hound.yaml diff --git a/doc/source/codesearch.rst b/doc/source/codesearch.rst index 34eaa76167..9e3f315033 100644 --- a/doc/source/codesearch.rst +++ b/doc/source/codesearch.rst @@ -5,31 +5,29 @@ Code Search ########### -The `Hound `_ code search engine is deployed in -our infrastructure to service all OpenStack repositories. +The `Hound `_ code search +engine is deployed in our infrastructure to service all OpenStack +repositories. At a Glance =========== :Hosts: - * http://codesearch.openstack.org + * http://codesearch.opendev.org :Puppet: - * https://opendev.org/opendev/puppet-hound - * :git_file:`modules/openstack_project/manifests/codesearch.pp` + * :git_file:`playbooks/roles/codesearch` :Projects: - * https://github.com/etsy/Hound + * https://github.com/hound-search/hound :Bugs: * https://storyboard.openstack.org/#!/project/748 - * https://github.com/etsy/Hound/issues :Resources: - * `Hound README `_ + * `Hound README `_ Overview ======== Hound is configured to read projects from a config.json file that is -automatically generated from the Gerrit projects.yaml, defined in the -$::project_config::jeepyb_project_file variable in Puppet. +automatically generated from the Gerrit projects.yaml Maintenance diff --git a/docker/hound/Dockerfile b/docker/hound/Dockerfile new file mode 100644 index 0000000000..76c2fb48a7 --- /dev/null +++ b/docker/hound/Dockerfile @@ -0,0 +1,37 @@ +# Copyright (c) 2020 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +FROM docker.io/opendevorg/python-base:3.8 + +ENV GOPATH /go + +RUN apt-get update \ + && apt-get install -y curl golang git + +RUN go get github.com/hound-search/hound/cmds/... + +RUN pip install git+https://opendev.org/opendev/jeepyb#egg=jeepyb \ + supervisor + +RUN apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +ADD supervisord.conf /etc/supervisor/conf.d/supervisord.conf +ADD start-container.sh /usr/bin/start-container +ADD update-hound-config.sh /usr/local/bin/update-hound-config +ADD resync-hound.sh /usr/local/bin/resync-hound +ADD sample-projects.yaml /var/run/sample-projects.yaml + +ENTRYPOINT ["start-container"] + diff --git a/docker/hound/resync-hound.sh b/docker/hound/resync-hound.sh new file mode 100755 index 0000000000..c209d39208 --- /dev/null +++ b/docker/hound/resync-hound.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +rc=0 + +update-hound-config || rc=$? + +if [[ ${rc} == 2 ]]; then + echo "No project modified" + exit 0 +elif [[ ${rc} == 0 ]]; then + echo "*** New projects found, restarting houndd" + supervisorctl restart houndd +else + echo "*** Unknown exit: ${rc}" + exit ${rc} +fi diff --git a/docker/hound/sample-projects.yaml b/docker/hound/sample-projects.yaml new file mode 100644 index 0000000000..ff3da1aaed --- /dev/null +++ b/docker/hound/sample-projects.yaml @@ -0,0 +1,15 @@ +- project: opendev/system-config + use-storyboard: true + groups: + - openstack-ci + description: System configuration for OpenStack Infrastructure +- project: openstack/project-config + use-storyboard: true + groups: + - openstack-ci + description: Configuration files for project CI systems +- project: zuul/zuul + use-storyboard: true + groups: + - zuul + description: The Gatekeeper, or a project gating system diff --git a/docker/hound/start-container.sh b/docker/hound/start-container.sh new file mode 100755 index 0000000000..33be187aa9 --- /dev/null +++ b/docker/hound/start-container.sh @@ -0,0 +1,9 @@ +#!/bin/sh +if [ $# -gt 0 ]; then + exec "$@" +else + if [ ! -f /var/run/config.json ]; then + update-hound-config; + fi + /usr/local/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf +fi diff --git a/docker/hound/supervisord.conf b/docker/hound/supervisord.conf new file mode 100644 index 0000000000..d19316792e --- /dev/null +++ b/docker/hound/supervisord.conf @@ -0,0 +1,19 @@ +[supervisord] +nodaemon = true + +[supervisorctl] + +[inet_http_server] +port = 127.0.0.1:9001 + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[program:houndd] +directory=/var/run +command=/go/bin/houndd -conf /var/run/config.json +logfile_maxbytes=0 +stdout_logfile_maxbytes=0 +stderr_logfile_maxbytes=0 +stdout_logfile=/dev/stdout +stderr_logfile=/dev/stdout diff --git a/docker/hound/update-hound-config.sh b/docker/hound/update-hound-config.sh new file mode 100755 index 0000000000..912322c05a --- /dev/null +++ b/docker/hound/update-hound-config.sh @@ -0,0 +1,34 @@ +#!/bin/bash -x + +CONFIG_DIR=/var/run +PROJECTS_FILE_NAME=projects.yaml +CONFIG_FILE_NAME=config.json +PROJECTS_FILE=${CONFIG_DIR}/${PROJECTS_FILE_NAME} +CONFIG_FILE=${CONFIG_DIR}/${CONFIG_FILE_NAME} + +PROJECT_CONFIG=https://opendev.org/openstack/project-config/raw/branch/master/gerrit/projects.yaml + +pushd $CONFIG_DIR + +# 2 signals nothing done, 0 means updated +_exit=2 + +if [ ${USE_HOUND_TEST_CONFIG:-} = 1 ]; then + PROJECTS_YAML=/var/run/sample-projects.yaml create-hound-config + exit 0 +fi + +curl -o ${PROJECTS_FILE}.tmp ${PROJECT_CONFIG} +md5sum ${PROJECTS_FILE}.tmp > ${PROJECTS_FILE}.tmp.md5 + +if [ ! -f ${PROJECTS_FILE} ] || \ + ! cmp --silent ${PROJECTS_FILE}.md5 ${PROJECTS_FILE}.tmp.md5; then + mv ${PROJECTS_FILE}.tmp ${PROJECTS_FILE} + mv ${PROJECTS_FILE}.tmp.md5 ${PROJECTS_FILE}.md5 + PROJECTS_YAML=${PROJECTS_FILE} create-hound-config + _exit=0 +fi + +popd + +exit $_exit diff --git a/inventory/base/hosts.yaml b/inventory/base/hosts.yaml index d17c1b2949..c1fe0f00ca 100644 --- a/inventory/base/hosts.yaml +++ b/inventory/base/hosts.yaml @@ -91,13 +91,6 @@ all: region_name: DFW public_v4: 172.99.116.215 public_v6: 2001:4800:7821:105:be76:4eff:fe04:b9a5 - codesearch01.openstack.org: - ansible_host: 23.253.92.77 - location: - cloud: openstackci-rax - region_name: DFW - public_v4: 23.253.92.77 - public_v6: 2001:4800:7815:105:be76:4eff:fe04:5fdf eavesdrop01.openstack.org: ansible_host: 104.130.124.113 location: diff --git a/inventory/service/groups.yaml b/inventory/service/groups.yaml index 0341c936c3..83320faecc 100644 --- a/inventory/service/groups.yaml +++ b/inventory/service/groups.yaml @@ -56,7 +56,7 @@ groups: cloud-launcher: - bridge.openstack.org codesearch: - - codesearch[0-9]*.open*.org + - codesearch[0-9]*.opendev.org control-plane-clouds: - bridge.openstack.org disabled: @@ -93,6 +93,7 @@ groups: kdc: - kdc[0-9]*.open*.org letsencrypt: + - codesearch[0-9]*.opendev.org - etherpad[0-9]*.opendev.org - gitea[0-9]*.opendev.org - graphite[0-9]*.opendev.org @@ -143,7 +144,6 @@ groups: - ask*.open*.org - backup[0-9]*.openstack.org - cacti[0-9]*.open*.org - - codesearch[0-9]*.open*.org - corvustest - eavesdrop[0-9]*.open*.org - elasticsearch[0-9]*.open*.org @@ -178,7 +178,6 @@ groups: - ask*.open*.org - ask-staging[0-9]*.open*.org - cacti[0-9]*.open*.org - - codesearch[0-9]*.open*.org - eavesdrop[0-9]*.open*.org - elasticsearch[0-9]*.open*.org - ethercalc[0-9]*.open*.org @@ -234,7 +233,7 @@ groups: webservers: - ask*.open*.org - cacti[0-9]*.open*.org - - codesearch[0-9]*.open*.org + - codesearch[0-9]*.opendev.org - eavesdrop[0-9]*.open*.org - ethercalc[0-9]*.open*.org - etherpad[0-9]*.open*.org diff --git a/inventory/service/host_vars/codesearch01.opendev.org.yaml b/inventory/service/host_vars/codesearch01.opendev.org.yaml new file mode 100644 index 0000000000..3492f653cd --- /dev/null +++ b/inventory/service/host_vars/codesearch01.opendev.org.yaml @@ -0,0 +1,5 @@ +letsencrypt_certs: + codesearch01-opendev-org-main: + - codesearch01.opendev.org + - codesearch.opendev.org + - codesearch.openstack.org diff --git a/manifests/codesearch.pp b/manifests/codesearch.pp deleted file mode 100644 index 41f45cdb9a..0000000000 --- a/manifests/codesearch.pp +++ /dev/null @@ -1,5 +0,0 @@ -# Node-OS: xenial -node /^codesearch\d*\.open.*\.org$/ { - $group = "codesearch" - class { 'openstack_project::codesearch': } -} diff --git a/modules/openstack_project/files/resync-hound-config.sh b/modules/openstack_project/files/resync-hound-config.sh deleted file mode 100644 index ec6c217682..0000000000 --- a/modules/openstack_project/files/resync-hound-config.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -# Copyright 2017 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -PROJECTS_YAML=${PROJECTS_YAML:-/etc/project-config/gerrit/projects.yaml} -REINDEX_LOCK=/var/www/hound/reindex.lock - -TEMP_DIR=$(mktemp -d) -trap "rm -rf ${TEMP_DIR} EXIT" - -pushd ${TEMP_DIR} - -echo $(date) -echo "Starting hound config update" - -# Generate the new config -PROJECTS_YAML=${PROJECTS_YAML} create-hound-config - -# See if we need to update -NEW="$(md5sum config.json | awk '{print $1}')" -OLD="$(md5sum /home/hound/config.json | awk '{print $1}')" -if [[ ${NEW} == ${OLD} ]]; then - echo "Nothing to do" - exit 0 -fi - -echo "Recreating config" - -# Move the new config into place -chown hound:hound config.json -chmod 0644 config.json -cp /home/hound/config.json /home/hound/config.json.bak -mv ./config.json /home/hound/config.json - -# release the hounds -touch ${REINDEX_LOCK} -service hound stop -sleep 2 -service hound start - -# Hound takes a few minutes to go through all our projects. We know -# it's ready when we see it listening on port 6080 -echo "Waiting for hound..." -while ! netstat -lnt | grep -q ':6080.*LISTEN\s*$' ; do - echo " ... still waiting" - sleep 5 -done - -rm ${REINDEX_LOCK} - -echo "... done" - diff --git a/modules/openstack_project/manifests/codesearch.pp b/modules/openstack_project/manifests/codesearch.pp deleted file mode 100644 index 8a5212e24b..0000000000 --- a/modules/openstack_project/manifests/codesearch.pp +++ /dev/null @@ -1,54 +0,0 @@ -# Class to configure hound on a node. -class openstack_project::codesearch { - - class { 'hound': - manage_config => false, - } - - include ::jeepyb - include ::logrotate - include ::pip - - file { '/home/hound/config.json': - ensure => 'present', - } - - file { '/usr/local/bin/resync-hound-config': - ensure => present, - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/openstack_project/resync-hound-config.sh', - } - - # Note: we could trigger this from project-config changes, but it - # does bring the service down for several minutes if something - # changes. Once a day should be enough. - cron { 'hound': - user => root, - hour => '4', - minute => '0', - command => 'flock -n /var/run/hound.sync.lock resync-hound-config >> /var/log/hound.sync.log 2>&1', - environment => [ - 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', - "PROJECTS_YAML=/opt/project-config/projects.yaml", - ], - require => [ - File['/usr/local/bin/resync-hound-config'], - File['/home/hound/config.json'], - ], - } - - logrotate::file { 'hound-sync': - log => '/var/log/hound.sync.log', - options => [ - 'compress', - 'copytruncate', - 'missingok', - 'rotate 7', - 'daily', - 'notifempty', - ], - } - -} diff --git a/playbooks/roles/codesearch/README.rst b/playbooks/roles/codesearch/README.rst new file mode 100644 index 0000000000..cac72a9156 --- /dev/null +++ b/playbooks/roles/codesearch/README.rst @@ -0,0 +1 @@ +Run a hound container to index Opendev code diff --git a/playbooks/roles/codesearch/defaults/main.yaml b/playbooks/roles/codesearch/defaults/main.yaml new file mode 100644 index 0000000000..55bb2b2c54 --- /dev/null +++ b/playbooks/roles/codesearch/defaults/main.yaml @@ -0,0 +1 @@ +codesearch_use_test_config: False diff --git a/playbooks/roles/codesearch/handlers/main.yaml b/playbooks/roles/codesearch/handlers/main.yaml new file mode 100644 index 0000000000..d24ae5182f --- /dev/null +++ b/playbooks/roles/codesearch/handlers/main.yaml @@ -0,0 +1,4 @@ +- name: codesearch Reload apache2 + service: + name: apache2 + state: reloaded diff --git a/playbooks/roles/codesearch/tasks/main.yaml b/playbooks/roles/codesearch/tasks/main.yaml new file mode 100644 index 0000000000..b9a71ac025 --- /dev/null +++ b/playbooks/roles/codesearch/tasks/main.yaml @@ -0,0 +1,78 @@ +- name: Ensure docker-compose directory exists + file: + state: directory + path: /etc/hound-docker + +- name: Write settings file + template: + src: docker-compose.yaml.j2 + dest: /etc/hound-docker/docker-compose.yaml + +- name: Install apache2 + apt: + name: + - apache2 + - apache2-utils + state: present + +- name: Apache modules + apache2_module: + state: present + name: "{{ item }}" + loop: + - rewrite + - proxy + - proxy_http + - ssl + - headers + - proxy_wstunnel + +- name: Copy apache config + template: + src: codesearch.vhost.j2 + dest: /etc/apache2/sites-enabled/000-default.conf + owner: root + group: root + mode: 0644 + notify: codesearch Reload apache2 + +- name: Create hound data storage area + file: + state: directory + path: /var/lib/hound/data + owner: root + group: root + mode: 0755 + +- name: Run docker-compose pull + shell: + cmd: docker-compose pull + chdir: /etc/hound-docker/ + +- name: Run docker-compose up + shell: + cmd: docker-compose up -d + chdir: /etc/hound-docker/ + +- name: Run docker prune to cleanup unneeded images + shell: + cmd: docker image prune -f + +# Daily update of codesearch. This only reloads hound +# if the project-config yaml has changed +- name: Install update cron job + cron: + name: Update codesearch + state: present + user: root + job: > + /usr/local/bin/docker-compose -f /etc/hound-docker/docker-compose.yaml exec -T hound + /usr/local/bin/resync-hound >> /var/log/resync-hound.log 2>&1 + hour: 5 + minute: 30 + +- name: Rotate sync logs + include_role: + name: logrotate + vars: + logrotate_file_name: /var/log/resync-hound.log diff --git a/playbooks/roles/codesearch/templates/codesearch.vhost.j2 b/playbooks/roles/codesearch/templates/codesearch.vhost.j2 new file mode 100644 index 0000000000..17549fa59a --- /dev/null +++ b/playbooks/roles/codesearch/templates/codesearch.vhost.j2 @@ -0,0 +1,41 @@ + + ServerName {{ inventory_hostname }} + ServerAdmin infra-root@openstack.org + + ErrorLog ${APACHE_LOG_DIR}/codesearch-error.log + + LogLevel warn + + CustomLog ${APACHE_LOG_DIR}/codesearch-access.log combined + + Redirect / https://codesearch.opendev.org/ + + + + + ServerName {{ inventory_hostname }} + ServerAdmin webmaster@openstack.org + + AllowEncodedSlashes On + + ErrorLog ${APACHE_LOG_DIR}/codesearch-ssl-error.log + + LogLevel warn + + CustomLog ${APACHE_LOG_DIR}/codesearch-ssl-access.log combined + + SSLEngine on + SSLProtocol All -SSLv2 -SSLv3 + # Note: this list should ensure ciphers that provide forward secrecy + SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP + SSLHonorCipherOrder on + + SSLCertificateFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.cer + SSLCertificateKeyFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.key + SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer + + ProxyPass / http://localhost:6080/ retry=0 + ProxyPassReverse / http://localhost:6080/ + + + diff --git a/playbooks/roles/codesearch/templates/docker-compose.yaml.j2 b/playbooks/roles/codesearch/templates/docker-compose.yaml.j2 new file mode 100644 index 0000000000..460eb83499 --- /dev/null +++ b/playbooks/roles/codesearch/templates/docker-compose.yaml.j2 @@ -0,0 +1,15 @@ +version: '3' + +services: + hound: + restart: always + image: docker.io/opendevorg/hound + network_mode: host + environment: + - 'USE_HOUND_TEST_CONFIG={{ "1" if codesearch_use_test_config else "0" }}' + volumes: + - /var/lib/hound/data:/var/run/data + logging: + driver: syslog + options: + tag: "docker-hound" diff --git a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml index 0d8a48c9f5..be04e2f520 100644 --- a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml +++ b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml @@ -120,6 +120,9 @@ - name: letsencrypt updated grafana01-opendev-org-main include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml +- name: letsencrypt updated codesearch01-opendev-org-main + include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml + # nodepool - name: letsencrypt updated nb01-opendev-org-main diff --git a/playbooks/service-codesearch.yaml b/playbooks/service-codesearch.yaml index fb6e7d9167..9a86ef49b5 100644 --- a/playbooks/service-codesearch.yaml +++ b/playbooks/service-codesearch.yaml @@ -1,15 +1,6 @@ -- hosts: 'localhost:!disabled' - name: Install puppet role/modules - strategy: linear - roles: - - puppet-setup-ansible - -- hosts: 'codesearch:!disabled' - name: "codesearch: run puppet on codesearch" - strategy: free +- hosts: "codesearch:!disabled" + name: "Configure codesearch" roles: - iptables - - sync-project-config - - pip3 - - name: puppet-run - manifest: /opt/system-config/production/manifests/codesearch.pp + - install-docker + - codesearch diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml index 44e7f51cd6..42d7bd1d16 100644 --- a/playbooks/zuul/run-base.yaml +++ b/playbooks/zuul/run-base.yaml @@ -69,6 +69,7 @@ - group_vars/zuul-scheduler.yaml - group_vars/zuul-web.yaml - host_vars/bridge.openstack.org.yaml + - host_vars/codesearch01.opendev.org.yaml - host_vars/etherpad01.opendev.org.yaml - host_vars/letsencrypt01.opendev.org.yaml - host_vars/letsencrypt02.opendev.org.yaml diff --git a/playbooks/zuul/templates/host_vars/codesearch01.opendev.org.yaml.j2 b/playbooks/zuul/templates/host_vars/codesearch01.opendev.org.yaml.j2 new file mode 100644 index 0000000000..d0186e9179 --- /dev/null +++ b/playbooks/zuul/templates/host_vars/codesearch01.opendev.org.yaml.j2 @@ -0,0 +1 @@ +codesearch_use_test_config: True diff --git a/testinfra/test_codesearch.py b/testinfra/test_codesearch.py new file mode 100644 index 0000000000..a057caf7cd --- /dev/null +++ b/testinfra/test_codesearch.py @@ -0,0 +1,27 @@ +# Copyright 2020 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +testinfra_hosts = ['codesearch01.opendev.org'] + + +def test_codesearch_container_listening(host): + codesearch = host.socket("tcp://127.0.0.1:6080") + assert codesearch.is_listening + +def test_codesearch_proxy(host): + cmd = host.run('curl --insecure ' + '--resolve codesearch.opendev.org:443:127.0.0.1 ' + 'https://codesearch.opendev.org') + assert 'Hound' in cmd.stdout diff --git a/zuul.d/docker-images/hound.yaml b/zuul.d/docker-images/hound.yaml new file mode 100644 index 0000000000..aa6d390d91 --- /dev/null +++ b/zuul.d/docker-images/hound.yaml @@ -0,0 +1,27 @@ +# Hound jobs +- job: + name: system-config-build-image-hound + description: Build a hound image. + provides: hound-container-image + parent: system-config-build-image + vars: &hound_vars + docker_images: + - context: docker/hound + repository: opendevorg/hound + files: &hound_files + - docker/hound/ + +- job: + name: system-config-upload-image-hound + description: Build and upload a hound image. + provides: hound-container-image + parent: system-config-upload-image + vars: *hound_vars + files: *hound_files + +- job: + name: system-config-promote-image-hound + description: Promote a previously published hound image to latest. + parent: system-config-promote-image + vars: *hound_vars + files: *hound_files diff --git a/zuul.d/infra-prod.yaml b/zuul.d/infra-prod.yaml index 72602d68e6..16070978ac 100644 --- a/zuul.d/infra-prod.yaml +++ b/zuul.d/infra-prod.yaml @@ -448,31 +448,6 @@ - docker/jinja-init/ - docker/python-base/ -- job: - name: infra-prod-service-codesearch - parent: infra-prod-service-base - description: Run service-codesearch.yaml playbook. - required-projects: - - opendev/ansible-role-puppet - - opendev/system-config - - openstack/project-config - vars: - playbook_name: service-codesearch.yaml - files: - - inventory/ - - playbooks/install-ansible.yaml - - playbooks/service-codesearch.yaml - - inventory/service/group_vars/puppet.yaml - - playbooks/roles/run-puppet/ - - playbooks/roles/install-ansible-roles/ - - playbooks/roles/iptables/ - - playbooks/roles/sync-project-config - - playbooks/roles/puppet-install/ - - playbooks/roles/disable-puppet-agent/ - - modules/openstack_project/manifests/codesearch.pp - - modules/openstack_project/files/resync-hound-config.sh - - manifests/codesearch.pp - - job: name: infra-prod-service-eavesdrop parent: infra-prod-service-base @@ -526,6 +501,24 @@ - playbooks/roles/accessbot - docker/accessbot/ +- job: + name: infra-prod-service-codesearch + parent: infra-prod-service-base + description: Run service-codesearch.yaml playbook. + vars: + playbook_name: service-codesearch.yaml + files: + - docker/hound/ + - inventory/ + - playbooks/service-codesearch.yaml + - inventory/service/host_vars/codesearch01.opendev.yaml + - inventory/service/group_vars/codesearch + - playbooks/roles/install-docker/ + - playbooks/roles/pip3/ + - playbooks/roles/codesearch + - playbooks/roles/logrotate + - playbooks/roles/iptables + - job: name: infra-prod-service-grafana parent: infra-prod-service-base diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 2bf2cccfd5..449d0aaa35 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -21,7 +21,11 @@ - name: opendev-buildset-registry - name: system-config-build-image-accessbot soft: true - - system-config-run-codesearch + - system-config-run-codesearch: + dependencies: + - name: opendev-buildset-registry + - name: system-config-build-image-hound + soft: true - system-config-run-lists - system-config-run-nodepool - system-config-run-meetpad: @@ -70,6 +74,11 @@ - name: opendev-buildset-registry - name: system-config-build-image-jinja-init soft: true + - system-config-build-image-hound: + dependencies: + - name: opendev-buildset-registry + - name: system-config-build-image-python-base-3.8 + soft: true - system-config-build-image-etherpad - system-config-build-image-gitea - system-config-build-image-grafana @@ -107,7 +116,11 @@ - name: opendev-buildset-registry - name: system-config-upload-image-accessbot soft: true - - system-config-run-codesearch + - system-config-run-codesearch: + dependencies: + - name: opendev-buildset-registry + - name: system-config-upload-image-hound + soft: true - system-config-run-lists - system-config-run-nodepool - system-config-run-meetpad: @@ -156,6 +169,7 @@ - name: opendev-buildset-registry - name: system-config-upload-image-jinja-init soft: true + - system-config-upload-image-hound - system-config-upload-image-etherpad - system-config-upload-image-gitea - system-config-upload-image-grafana @@ -181,6 +195,7 @@ - opendev-promote-docs deploy: jobs: + - system-config-promote-image-hound - system-config-promote-image-jinja-init - system-config-promote-image-gitea-init - system-config-promote-image-gitea @@ -218,6 +233,12 @@ - infra-prod-service-gitea-lb - infra-prod-service-nameserver - infra-prod-service-nodepool + - infra-prod-service-codesearch: + dependencies: + - name: infra-prod-letsencrypt + soft: true + - name: system-config-promote-image-hound + soft: true - infra-prod-service-etherpad: dependencies: - name: infra-prod-install-ansible diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index e9a9fc9ff8..a2c64ddd1d 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -156,37 +156,6 @@ - docker/accessbot/ - testinfra/test_eavesdrop.py -- job: - name: system-config-run-codesearch - parent: system-config-run - description: | - Run the playbook for an codesearch server. - nodeset: - nodes: - - name: bridge.openstack.org - label: ubuntu-bionic - - name: codesearch01.openstack.org - label: ubuntu-xenial - required-projects: - - opendev/ansible-role-puppet - - opendev/system-config - - openstack/project-config - files: - - playbooks/install-ansible.yaml - - playbooks/service-codesearch.yaml - - inventory/service/group_vars/puppet.yaml - - playbooks/roles/run-puppet/ - - playbooks/roles/install-ansible-roles/ - - playbooks/roles/sync-project-config - - playbooks/roles/puppet-install/ - - playbooks/roles/disable-puppet-agent/ - - modules/openstack_project/manifests/codesearch.pp - - modules/openstack_project/files/resync-hound-config.sh - - manifests/codesearch.pp - vars: - run_playbooks: - - playbooks/service-codesearch.yaml - - job: name: system-config-run-letsencrypt parent: system-config-run @@ -501,6 +470,35 @@ - playbooks/roles/install-docker/ - testinfra/test_registry.py +- job: + name: system-config-run-codesearch + parent: system-config-run-containers + description: | + Run the playbook for the codesearch server. + timeout: 3600 + requires: codesearch-container-image + required-projects: + - opendev/system-config + nodeset: + nodes: + - name: bridge.openstack.org + label: ubuntu-bionic + - name: codesearch01.opendev.org + label: ubuntu-focal + vars: + run_playbooks: + - playbooks/letsencrypt.yaml + - playbooks/service-codesearch.yaml + files: + - playbooks/bridge.yaml + - playbooks/letsencrypt.yaml + - playbooks/service-codesearch.yaml + - playbooks/roles/codesearch/ + - playbooks/roles/install-docker/ + - playbooks/roles/pip3/ + - docker/codesearch/ + - testinfra/test_codesearch.py + - job: name: system-config-run-etherpad parent: system-config-run-containers