Deploy refstack with ansible docker

This adds a dockerfile to build an opendevorg/refstack image as well as
the jobs to build and publish it.

Change-Id: Icade6c713fa9bf6ab508fd4d8d65debada2ddb30
This commit is contained in:
Clark Boylan 2020-01-31 09:27:34 -08:00 committed by Martin Kopec
parent 326e0dc4d7
commit a4604ae0b3
21 changed files with 434 additions and 4 deletions

View File

@ -0,0 +1,55 @@
# Copyright (c) 2020 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM opendevorg/python-builder as builder
RUN apt-get update \
&& apt-get -y install git apt-utils python3-dev dh-python libc-dev-bin \
libc6-dev libexpat1-dev libpython3-dev libpython3-dev linux-libc-dev \
apt-transport-https curl gnupg2 \
&& curl -sS https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
&& echo "deb https://deb.nodesource.com/node_15.x bionic main" | tee /etc/apt/sources.list.d/nodesource.list \
&& curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
&& echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
&& apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get -q --option "Dpkg::Options::=--force-confold" --assume-yes install nodejs yarn \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
RUN git clone https://opendev.org/osf/refstack /tmp/src
RUN assemble
RUN cd /tmp/src && yarn install
# Refstack's docs are built and then hosted by refstack UI
RUN python -m venv /tmp/venv \
&& /tmp/venv/bin/pip install beautifulsoup4 docutils \
&& /tmp/venv/bin/python /tmp/src/tools/convert-docs.py -o /tmp/src/refstack-ui/app/components/about/templates /tmp/src/doc/source/*.rst \
&& rm -rf /tmp/venv
FROM opendevorg/python-base as refstack
COPY --from=builder /output/ /output
COPY --from=builder /tmp/src/refstack-ui/app/ /refstack-ui/app
COPY ./entrypoint.sh /usr/bin/entrypoint
# TODO this should be fixed probably through proper js packaging
RUN rm /refstack-ui/app/assets/lib
COPY --from=builder /tmp/src/node_modules/@bower_components/ /refstack-ui/app/assets/lib
RUN /output/install-from-bindep \
&& rm -rf /output
ENTRYPOINT ["/usr/bin/entrypoint"]
CMD ["pecan", "serve", "/usr/local/lib/python3.7/site-packages/refstack/api/config.py"]

4
docker/refstack/entrypoint.sh Executable file
View File

@ -0,0 +1,4 @@
#!/bin/bash
set -e
refstack-manage upgrade --revision head
$@

View File

@ -110,6 +110,7 @@ groups:
- review-test.opendev.org
- static[0-9]*.opendev.org
- zuul[0-9]*.open*.org
- refstack[0-9]*.openstack.org
logstash:
- logstash[0-9]*.open*.org
logstash-worker:
@ -165,7 +166,7 @@ groups:
- paste[0-9]*.open*.org
- pbx[0-9]*.opendev.org
- planet[0-9]*.open*.org
- refstack*.open*.org
- refstack.openstack.org
- status*.open*.org
- storyboard-dev[0-9]*.opendev.org
- storyboard[0-9]*.opendev.org
@ -198,7 +199,7 @@ groups:
- paste[0-9]*.open*.org
- pbx[0-9]*.opendev.org
- planet[0-9]*.open*.org
- refstack*.open*.org
- refstack.openstack.org
- status*.open*.org
- storyboard[0-9]*.opendev.org
- storyboard-dev[0-9]*.opendev.org
@ -209,7 +210,9 @@ groups:
- wiki[0-9]*.openstack.org
- wiki-dev[0-9]*.openstack.org
refstack:
- refstack*.open*.org
- refstack[0-9]*.openstack.org
refstack-docker:
- refstack[0-9]*.openstack.org
registry:
- insecure-ci-registry[0-9]*.opendev.org
review-dev:
@ -250,7 +253,7 @@ groups:
- openstackid[0-9]*.openstack.org
- paste[0-9]*.open*.org
- planet[0-9]*.open*.org
- refstack*.open*.org
- refstack*.openstack.org
- static[0-9]*.opendev.org
- status*.open*.org
- storyboard-dev[0-9]*.opendev.org

View File

@ -0,0 +1,4 @@
letsencrypt_certs:
refstack01-openstack-org-main:
- refstack01.openstack.org
- refstack.openstack.org

View File

@ -149,6 +149,11 @@
- name: letsencrypt updated review-test-opendev-org-main
include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
# refstack
- name: letsencrypt updated refstack01-openstack-org-main
include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
# Mirrors
- name: letsencrypt updated mirror01-dfw-rax-main

View File

@ -0,0 +1 @@
Install, configure, and run a refstack server.

View File

@ -0,0 +1,2 @@
User-agent: *
Disallow: /

View File

@ -0,0 +1,4 @@
- name: refstack Reload apache2
service:
name: apache2
state: reloaded

View File

@ -0,0 +1,121 @@
- name: Ensure docker-compose directory exists
file:
state: directory
path: /etc/refstack-docker
mode: 0700
- name: Write docker-compose file
template:
src: docker-compose.yaml.j2
dest: /etc/refstack-docker/docker-compose.yaml
mode: 0600
- name: Install apache2
apt:
name:
- apache2
- apache2-utils
state: present
- name: Apache modules
apache2_module:
state: present
name: "{{ item }}"
loop:
- rewrite
- proxy
- proxy_http
- ssl
- headers
- proxy_wstunnel
- name: Copy apache config
template:
src: refstack.vhost.j2
dest: /etc/apache2/sites-enabled/000-default.conf
owner: root
group: root
mode: 0644
notify: refstack Reload apache2
- name: Create refstack data storage area
file:
state: directory
path: /var/lib/refstack/data
owner: root
group: root
mode: 0755
- name: Create refstack www storage area
file:
state: directory
path: /var/lib/refstack/www
owner: root
group: root
mode: 0755
- name: Copy hound robots.txt
copy:
src: robots.txt
dest: /var/lib/refstack/www/robots.txt
- name: Ensure refstack volume directory exists
file:
state: directory
path: "/var/refstack"
- name: Write refstack.conf
template:
src: refstack.conf.j2
dest: /var/refstack/refstack.conf
- name: Write config.json
template:
src: config.json.j2
dest: /var/refstack/config.json
- name: Install distro packages
package:
name:
- docker-compose
state: present
- name: Run docker-compose pull
shell:
cmd: docker-compose pull
chdir: /etc/refstack-docker/
- name: Run docker-compose up
shell:
cmd: docker-compose up -d --timeout 60
chdir: /etc/refstack-docker/
- name: Run docker prune to cleanup unneeded images
shell:
cmd: docker image prune -f
- name: Create db backup dest
file:
state: directory
path: /var/backups/refstack-mariadb
mode: 0700
owner: root
group: root
- name: Set up cron job to backup the database
cron:
name: refstack-db-backup
state: present
user: root
job: >
/usr/bin/docker-compose -f /etc/refstack-docker/docker-compose.yaml exec -T mariadb
bash -c '/usr/bin/mysqldump --opt --databases refstack --single-transaction -uroot -p"$MYSQL_ROOT_PASSWORD"' |
gzip -9 > /var/backups/refstack-mariadb/refstack-mariadb.sql.gz
minute: "42"
hour: "4"
- name: Rotate db backups
include_role:
name: logrotate
vars:
logrotate_file_name: /var/backups/refstack-mariadb/refstack-mariadb.sql.gz

View File

@ -0,0 +1 @@
{"refstackApiUrl": "{{ refstack_url }}/v1"}

View File

@ -0,0 +1,24 @@
# Version 2 is the latest that is supported by docker-compose in
# Ubuntu Xenial.
version: '2'
services:
mariadb:
image: docker.io/library/mariadb:10.4
network_mode: host
restart: always
environment:
MYSQL_ROOT_PASSWORD: "{{ refstack_root_db_password }}"
MYSQL_DATABASE: refstack
MYSQL_USER: "{{ refstack_db_username }}"
MYSQL_PASSWORD: "{{ refstack_db_password }}"
refstack-api:
depends_on:
- mariadb
image: opendevorg/refstack:latest
network_mode: host
restart: always
volumes:
- /var/refstack/refstack.conf:/etc/refstack.conf
- /var/refstack/config.json:/refstack-ui/app/config.json
- /var/lib/refstack/data:/var/run/data

View File

@ -0,0 +1,17 @@
[DEFAULT]
debug = true
verbose = true
ui_url = {{ refstack_url }}
[api]
static_root = /refstack-ui/app
template_path = /refstack-ui/app
app_dev_mode = true
api_url = {{ refstack_url }}
enable_anonymous_upload = {{ enable_anonymous_upload | default(false) }}
[database]
connection = "mysql+pymysql://{{ refstack_db_username}}:{{ refstack_db_password }}@localhost/refstack?charset=utf8"
[osid]
openstack_openid_endpoint = {{ refstack_openid_endpoint }}

View File

@ -0,0 +1,53 @@
<VirtualHost *:80>
ServerName {{ inventory_hostname }}
ServerAdmin infra-root@openstack.org
ErrorLog ${APACHE_LOG_DIR}/refstack-error.log
LogLevel warn
CustomLog ${APACHE_LOG_DIR}/refstack-access.log combined
Redirect / https://refstack.openstack.org/
</VirtualHost>
<VirtualHost *:443>
ServerName {{ inventory_hostname }}
ServerAdmin webmaster@openstack.org
RewriteCond %{HTTP_HOST} !^refstack\.openstack\.org [nocase]
RewriteRule ^/(.*) https://refstack.openstack.org/$1 [last,redirect=permanent]
AllowEncodedSlashes On
ErrorLog ${APACHE_LOG_DIR}/refstack-ssl-error.log
LogLevel warn
CustomLog ${APACHE_LOG_DIR}/refstack-ssl-access.log combined
SSLEngine on
SSLProtocol All -SSLv2 -SSLv3
# Note: this list should ensure ciphers that provide forward secrecy
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
SSLHonorCipherOrder on
SSLCertificateFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.cer
SSLCertificateKeyFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.key
SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer
ProxyPass / http://localhost:8000/ retry=0
ProxyPassReverse / http://localhost:8000/
<Location "/robots.txt">
ProxyPass !
</Location>
<Directory "/var/lib/refstack/www">
Require all granted
</Directory>
Alias /robots.txt /var/lib/refstack/www/robots.txt
</VirtualHost>

View File

@ -0,0 +1,5 @@
- hosts: "refstack-docker:!disabled"
name: "Configure refstack service in docker"
roles:
- install-docker
- refstack

View File

@ -62,6 +62,7 @@
- group_vars/meetpad.yaml
- group_vars/jvb.yaml
- group_vars/nodepool-launcher.yaml
- group_vars/refstack-docker.yaml
- group_vars/registry.yaml
- group_vars/review.yaml
- group_vars/control-plane-clouds.yaml
@ -84,6 +85,7 @@
- host_vars/mirror-update01.opendev.org.yaml
- host_vars/backup-test01.opendev.org.yaml
- host_vars/backup-test02.opendev.org.yaml
- host_vars/refstack01.openstack.org.yaml
- name: Display group membership
command: ansible localhost -m debug -a 'var=groups'
- name: Run base.yaml

View File

@ -0,0 +1,5 @@
refstack_url: http://{{ ansible_fqdn }}:8000
refstack_db_username: refstack
refstack_db_password: Jz4ooq9TL7nc3hX3
refstack_root_db_password: KbgY3r9HYnEYpgRP
refstack_openid_endpoint: https://openstackid.org

View File

@ -0,0 +1 @@
enable_anonymous_upload: true

View File

@ -0,0 +1,58 @@
# Copyright 2020 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
import time
import urllib3
testinfra_hosts = ['refstack01.openstack.org:8000']
test_result_json = {
"cpid": "9cddf99456964d7c90b98362e7175a12",
"duration_seconds": 10,
"results": [
{
"name": "tempest.api.compute.flavors.test_flavors."
"FlavorsV2TestJSON.test_list_flavors",
"uuid": "e36c0eaa-dff5-4082-ad1f-3f9a80aa3f59"
}
]
}
def test_refstack_listening(host):
# Give it some time to come up
for i in range(300):
refstack_https = host.socket("tcp://0.0.0.0:443")
if refstack_https.is_listening:
break
time.sleep(1)
assert refstack_https.is_listening
refstack_http = host.socket("tcp://0.0.0.0:8000")
assert refstack_http.is_listening
def test_refstack_container_running(host):
cmd = host.run("docker inspect refstack-docker_refstack-api_1")
out = json.loads(cmd.stdout)
assert out[0]["State"]["Status"] == "running"
assert out[0]["RestartCount"] == 0
def test_result_submission(host):
url = testinfra_hosts[0] + "/v1/results/"
headers = {'Content-type': 'application/json'}
data = json.dumps(test_result_json)
http = urllib3.PoolManager()
resp = http.request('POST', url, body=data, headers=headers)
assert resp.status == 201

View File

@ -0,0 +1,25 @@
# Refstack jobs
- job:
name: system-config-build-image-refstack
description: Build a refstack image.
parent: system-config-build-image
vars: &refstack_vars
docker_images:
- context: docker/refstack
target: refstack
repository: opendevorg/refstack
# Duplicate in the run-refstack job
files: &refstack_files
- docker/refstack/.*
- job:
name: system-config-upload-image-refstack
description: Build and upload a refstack image.
parent: system-config-upload-image
vars: *refstack_vars
files: *refstack_files
- job:
name: system-config-promote-image-refstack
description: Promote a previously published refstack image to latest.
parent: system-config-promote-image
vars: *refstack_vars
files: *refstack_files

View File

@ -60,6 +60,11 @@
- name: opendev-buildset-registry
- name: system-config-build-image-gerrit-3.2
soft: true
- system-config-build-image-refstack
- system-config-run-refstack:
dependencies:
- name: system-config-build-image-refstack
soft: true
- system-config-run-zookeeper
- system-config-run-zuul
- system-config-run-zuul-preview

View File

@ -889,3 +889,38 @@
'/var/log/acme.sh/': logs
'/etc/apache2/': logs
'/var/log/apache2/': logs
- job:
name: system-config-run-refstack
parent: system-config-run
ansible-version: 2.9
description: |
Run the playbook for refstack server.
dependencies: opendev-buildset-registry
timeout: 3600
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: refstack01.openstack.org
label: ubuntu-focal
host-vars:
refstack01.openstack.org:
host_copy_output:
'/var/lib/refstack/': logs
'/var/refstack/': logs
vars:
run_playbooks:
- playbooks/letsencrypt.yaml
- playbooks/service-refstack.yaml
container_command: docker
files:
- playbooks/bridge.yaml
- playbooks/group_vars/refstack.yaml
- playbooks/host_vars/gitea.*
- playbooks/zuul/templates/group_vars/refstack.yaml.j2
- playbooks/roles/refstack/
- playbooks/roles/letsencrypt-create-certs/handlers/restart_apache.yaml
- testinfra/test_refstack.py
# If we rebuild the image, we want to run this job as well.
- docker/refstack/.*