diff --git a/docker/refstack/Dockerfile b/docker/refstack/Dockerfile
new file mode 100644
index 0000000000..4e189a29fa
--- /dev/null
+++ b/docker/refstack/Dockerfile
@@ -0,0 +1,55 @@
+# Copyright (c) 2020 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM opendevorg/python-builder as builder
+
+RUN apt-get update \
+ && apt-get -y install git apt-utils python3-dev dh-python libc-dev-bin \
+ libc6-dev libexpat1-dev libpython3-dev libpython3-dev linux-libc-dev \
+ apt-transport-https curl gnupg2 \
+ && curl -sS https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
+ && echo "deb https://deb.nodesource.com/node_15.x bionic main" | tee /etc/apt/sources.list.d/nodesource.list \
+ && curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
+ && echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
+ && apt-get update \
+ && DEBIAN_FRONTEND=noninteractive apt-get -q --option "Dpkg::Options::=--force-confold" --assume-yes install nodejs yarn \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN git clone https://opendev.org/osf/refstack /tmp/src
+
+RUN assemble
+
+RUN cd /tmp/src && yarn install
+
+# Refstack's docs are built and then hosted by refstack UI
+RUN python -m venv /tmp/venv \
+ && /tmp/venv/bin/pip install beautifulsoup4 docutils \
+ && /tmp/venv/bin/python /tmp/src/tools/convert-docs.py -o /tmp/src/refstack-ui/app/components/about/templates /tmp/src/doc/source/*.rst \
+ && rm -rf /tmp/venv
+
+FROM opendevorg/python-base as refstack
+
+COPY --from=builder /output/ /output
+COPY --from=builder /tmp/src/refstack-ui/app/ /refstack-ui/app
+COPY ./entrypoint.sh /usr/bin/entrypoint
+# TODO this should be fixed probably through proper js packaging
+RUN rm /refstack-ui/app/assets/lib
+COPY --from=builder /tmp/src/node_modules/@bower_components/ /refstack-ui/app/assets/lib
+RUN /output/install-from-bindep \
+ && rm -rf /output
+
+ENTRYPOINT ["/usr/bin/entrypoint"]
+CMD ["pecan", "serve", "/usr/local/lib/python3.7/site-packages/refstack/api/config.py"]
diff --git a/docker/refstack/entrypoint.sh b/docker/refstack/entrypoint.sh
new file mode 100755
index 0000000000..a701d07ea8
--- /dev/null
+++ b/docker/refstack/entrypoint.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+refstack-manage upgrade --revision head
+$@
diff --git a/inventory/service/groups.yaml b/inventory/service/groups.yaml
index b21282bf12..47efe54cb0 100644
--- a/inventory/service/groups.yaml
+++ b/inventory/service/groups.yaml
@@ -110,6 +110,7 @@ groups:
- review-test.opendev.org
- static[0-9]*.opendev.org
- zuul[0-9]*.open*.org
+ - refstack[0-9]*.openstack.org
logstash:
- logstash[0-9]*.open*.org
logstash-worker:
@@ -165,7 +166,7 @@ groups:
- paste[0-9]*.open*.org
- pbx[0-9]*.opendev.org
- planet[0-9]*.open*.org
- - refstack*.open*.org
+ - refstack.openstack.org
- status*.open*.org
- storyboard-dev[0-9]*.opendev.org
- storyboard[0-9]*.opendev.org
@@ -198,7 +199,7 @@ groups:
- paste[0-9]*.open*.org
- pbx[0-9]*.opendev.org
- planet[0-9]*.open*.org
- - refstack*.open*.org
+ - refstack.openstack.org
- status*.open*.org
- storyboard[0-9]*.opendev.org
- storyboard-dev[0-9]*.opendev.org
@@ -209,7 +210,9 @@ groups:
- wiki[0-9]*.openstack.org
- wiki-dev[0-9]*.openstack.org
refstack:
- - refstack*.open*.org
+ - refstack[0-9]*.openstack.org
+ refstack-docker:
+ - refstack[0-9]*.openstack.org
registry:
- insecure-ci-registry[0-9]*.opendev.org
review-dev:
@@ -250,7 +253,7 @@ groups:
- openstackid[0-9]*.openstack.org
- paste[0-9]*.open*.org
- planet[0-9]*.open*.org
- - refstack*.open*.org
+ - refstack*.openstack.org
- static[0-9]*.opendev.org
- status*.open*.org
- storyboard-dev[0-9]*.opendev.org
diff --git a/inventory/service/host_vars/refstack01.openstack.org.yaml b/inventory/service/host_vars/refstack01.openstack.org.yaml
new file mode 100644
index 0000000000..c4bbc4f8e1
--- /dev/null
+++ b/inventory/service/host_vars/refstack01.openstack.org.yaml
@@ -0,0 +1,4 @@
+letsencrypt_certs:
+ refstack01-openstack-org-main:
+ - refstack01.openstack.org
+ - refstack.openstack.org
diff --git a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml
index be04e2f520..abf40c848b 100644
--- a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml
+++ b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml
@@ -149,6 +149,11 @@
- name: letsencrypt updated review-test-opendev-org-main
include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+# refstack
+
+- name: letsencrypt updated refstack01-openstack-org-main
+ include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+
# Mirrors
- name: letsencrypt updated mirror01-dfw-rax-main
diff --git a/playbooks/roles/refstack/README.rst b/playbooks/roles/refstack/README.rst
new file mode 100644
index 0000000000..cdffcb12bb
--- /dev/null
+++ b/playbooks/roles/refstack/README.rst
@@ -0,0 +1 @@
+Install, configure, and run a refstack server.
diff --git a/playbooks/roles/refstack/files/robots.txt b/playbooks/roles/refstack/files/robots.txt
new file mode 100644
index 0000000000..1f53798bb4
--- /dev/null
+++ b/playbooks/roles/refstack/files/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /
diff --git a/playbooks/roles/refstack/handlers/main.yaml b/playbooks/roles/refstack/handlers/main.yaml
new file mode 100644
index 0000000000..2ebc74c3c8
--- /dev/null
+++ b/playbooks/roles/refstack/handlers/main.yaml
@@ -0,0 +1,4 @@
+- name: refstack Reload apache2
+ service:
+ name: apache2
+ state: reloaded
diff --git a/playbooks/roles/refstack/tasks/main.yaml b/playbooks/roles/refstack/tasks/main.yaml
new file mode 100644
index 0000000000..420386623a
--- /dev/null
+++ b/playbooks/roles/refstack/tasks/main.yaml
@@ -0,0 +1,121 @@
+- name: Ensure docker-compose directory exists
+ file:
+ state: directory
+ path: /etc/refstack-docker
+ mode: 0700
+
+- name: Write docker-compose file
+ template:
+ src: docker-compose.yaml.j2
+ dest: /etc/refstack-docker/docker-compose.yaml
+ mode: 0600
+
+- name: Install apache2
+ apt:
+ name:
+ - apache2
+ - apache2-utils
+ state: present
+
+- name: Apache modules
+ apache2_module:
+ state: present
+ name: "{{ item }}"
+ loop:
+ - rewrite
+ - proxy
+ - proxy_http
+ - ssl
+ - headers
+ - proxy_wstunnel
+
+- name: Copy apache config
+ template:
+ src: refstack.vhost.j2
+ dest: /etc/apache2/sites-enabled/000-default.conf
+ owner: root
+ group: root
+ mode: 0644
+ notify: refstack Reload apache2
+
+- name: Create refstack data storage area
+ file:
+ state: directory
+ path: /var/lib/refstack/data
+ owner: root
+ group: root
+ mode: 0755
+
+- name: Create refstack www storage area
+ file:
+ state: directory
+ path: /var/lib/refstack/www
+ owner: root
+ group: root
+ mode: 0755
+
+- name: Copy hound robots.txt
+ copy:
+ src: robots.txt
+ dest: /var/lib/refstack/www/robots.txt
+
+- name: Ensure refstack volume directory exists
+ file:
+ state: directory
+ path: "/var/refstack"
+
+- name: Write refstack.conf
+ template:
+ src: refstack.conf.j2
+ dest: /var/refstack/refstack.conf
+
+- name: Write config.json
+ template:
+ src: config.json.j2
+ dest: /var/refstack/config.json
+
+- name: Install distro packages
+ package:
+ name:
+ - docker-compose
+ state: present
+
+- name: Run docker-compose pull
+ shell:
+ cmd: docker-compose pull
+ chdir: /etc/refstack-docker/
+
+- name: Run docker-compose up
+ shell:
+ cmd: docker-compose up -d --timeout 60
+ chdir: /etc/refstack-docker/
+
+- name: Run docker prune to cleanup unneeded images
+ shell:
+ cmd: docker image prune -f
+
+- name: Create db backup dest
+ file:
+ state: directory
+ path: /var/backups/refstack-mariadb
+ mode: 0700
+ owner: root
+ group: root
+
+- name: Set up cron job to backup the database
+ cron:
+ name: refstack-db-backup
+ state: present
+ user: root
+ job: >
+ /usr/bin/docker-compose -f /etc/refstack-docker/docker-compose.yaml exec -T mariadb
+ bash -c '/usr/bin/mysqldump --opt --databases refstack --single-transaction -uroot -p"$MYSQL_ROOT_PASSWORD"' |
+ gzip -9 > /var/backups/refstack-mariadb/refstack-mariadb.sql.gz
+ minute: "42"
+ hour: "4"
+
+- name: Rotate db backups
+ include_role:
+ name: logrotate
+ vars:
+ logrotate_file_name: /var/backups/refstack-mariadb/refstack-mariadb.sql.gz
diff --git a/playbooks/roles/refstack/templates/config.json.j2 b/playbooks/roles/refstack/templates/config.json.j2
new file mode 100644
index 0000000000..ac79e2c242
--- /dev/null
+++ b/playbooks/roles/refstack/templates/config.json.j2
@@ -0,0 +1 @@
+{"refstackApiUrl": "{{ refstack_url }}/v1"}
diff --git a/playbooks/roles/refstack/templates/docker-compose.yaml.j2 b/playbooks/roles/refstack/templates/docker-compose.yaml.j2
new file mode 100644
index 0000000000..3b6e5ba914
--- /dev/null
+++ b/playbooks/roles/refstack/templates/docker-compose.yaml.j2
@@ -0,0 +1,24 @@
+# Version 2 is the latest that is supported by docker-compose in
+# Ubuntu Xenial.
+version: '2'
+
+services:
+ mariadb:
+ image: docker.io/library/mariadb:10.4
+ network_mode: host
+ restart: always
+ environment:
+ MYSQL_ROOT_PASSWORD: "{{ refstack_root_db_password }}"
+ MYSQL_DATABASE: refstack
+ MYSQL_USER: "{{ refstack_db_username }}"
+ MYSQL_PASSWORD: "{{ refstack_db_password }}"
+ refstack-api:
+ depends_on:
+ - mariadb
+ image: opendevorg/refstack:latest
+ network_mode: host
+ restart: always
+ volumes:
+ - /var/refstack/refstack.conf:/etc/refstack.conf
+ - /var/refstack/config.json:/refstack-ui/app/config.json
+ - /var/lib/refstack/data:/var/run/data
diff --git a/playbooks/roles/refstack/templates/refstack.conf.j2 b/playbooks/roles/refstack/templates/refstack.conf.j2
new file mode 100644
index 0000000000..7b7669079f
--- /dev/null
+++ b/playbooks/roles/refstack/templates/refstack.conf.j2
@@ -0,0 +1,17 @@
+[DEFAULT]
+debug = true
+verbose = true
+ui_url = {{ refstack_url }}
+
+[api]
+static_root = /refstack-ui/app
+template_path = /refstack-ui/app
+app_dev_mode = true
+api_url = {{ refstack_url }}
+enable_anonymous_upload = {{ enable_anonymous_upload | default(false) }}
+
+[database]
+connection = "mysql+pymysql://{{ refstack_db_username}}:{{ refstack_db_password }}@localhost/refstack?charset=utf8"
+
+[osid]
+openstack_openid_endpoint = {{ refstack_openid_endpoint }}
diff --git a/playbooks/roles/refstack/templates/refstack.vhost.j2 b/playbooks/roles/refstack/templates/refstack.vhost.j2
new file mode 100644
index 0000000000..cdbbc24ab7
--- /dev/null
+++ b/playbooks/roles/refstack/templates/refstack.vhost.j2
@@ -0,0 +1,53 @@
+
+ ServerName {{ inventory_hostname }}
+ ServerAdmin infra-root@openstack.org
+
+ ErrorLog ${APACHE_LOG_DIR}/refstack-error.log
+
+ LogLevel warn
+
+ CustomLog ${APACHE_LOG_DIR}/refstack-access.log combined
+
+ Redirect / https://refstack.openstack.org/
+
+
+
+
+ ServerName {{ inventory_hostname }}
+ ServerAdmin webmaster@openstack.org
+
+ RewriteCond %{HTTP_HOST} !^refstack\.openstack\.org [nocase]
+ RewriteRule ^/(.*) https://refstack.openstack.org/$1 [last,redirect=permanent]
+
+ AllowEncodedSlashes On
+
+ ErrorLog ${APACHE_LOG_DIR}/refstack-ssl-error.log
+
+ LogLevel warn
+
+ CustomLog ${APACHE_LOG_DIR}/refstack-ssl-access.log combined
+
+ SSLEngine on
+ SSLProtocol All -SSLv2 -SSLv3
+ # Note: this list should ensure ciphers that provide forward secrecy
+ SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
+ SSLHonorCipherOrder on
+
+ SSLCertificateFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.cer
+ SSLCertificateKeyFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.key
+ SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer
+
+ ProxyPass / http://localhost:8000/ retry=0
+ ProxyPassReverse / http://localhost:8000/
+
+
+ ProxyPass !
+
+
+ Require all granted
+
+ Alias /robots.txt /var/lib/refstack/www/robots.txt
+
+
+
+
diff --git a/playbooks/service-refstack.yaml b/playbooks/service-refstack.yaml
new file mode 100644
index 0000000000..68a76d9650
--- /dev/null
+++ b/playbooks/service-refstack.yaml
@@ -0,0 +1,5 @@
+- hosts: "refstack-docker:!disabled"
+ name: "Configure refstack service in docker"
+ roles:
+ - install-docker
+ - refstack
diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml
index f660335124..50aa0e6603 100644
--- a/playbooks/zuul/run-base.yaml
+++ b/playbooks/zuul/run-base.yaml
@@ -62,6 +62,7 @@
- group_vars/meetpad.yaml
- group_vars/jvb.yaml
- group_vars/nodepool-launcher.yaml
+ - group_vars/refstack-docker.yaml
- group_vars/registry.yaml
- group_vars/review.yaml
- group_vars/control-plane-clouds.yaml
@@ -84,6 +85,7 @@
- host_vars/mirror-update01.opendev.org.yaml
- host_vars/backup-test01.opendev.org.yaml
- host_vars/backup-test02.opendev.org.yaml
+ - host_vars/refstack01.openstack.org.yaml
- name: Display group membership
command: ansible localhost -m debug -a 'var=groups'
- name: Run base.yaml
diff --git a/playbooks/zuul/templates/group_vars/refstack-docker.yaml.j2 b/playbooks/zuul/templates/group_vars/refstack-docker.yaml.j2
new file mode 100644
index 0000000000..2954b9d534
--- /dev/null
+++ b/playbooks/zuul/templates/group_vars/refstack-docker.yaml.j2
@@ -0,0 +1,5 @@
+refstack_url: http://{{ ansible_fqdn }}:8000
+refstack_db_username: refstack
+refstack_db_password: Jz4ooq9TL7nc3hX3
+refstack_root_db_password: KbgY3r9HYnEYpgRP
+refstack_openid_endpoint: https://openstackid.org
diff --git a/playbooks/zuul/templates/host_vars/refstack01.openstack.org.yaml.j2 b/playbooks/zuul/templates/host_vars/refstack01.openstack.org.yaml.j2
new file mode 100644
index 0000000000..e42664697f
--- /dev/null
+++ b/playbooks/zuul/templates/host_vars/refstack01.openstack.org.yaml.j2
@@ -0,0 +1 @@
+enable_anonymous_upload: true
diff --git a/testinfra/test_refstack.py b/testinfra/test_refstack.py
new file mode 100644
index 0000000000..b126f02d45
--- /dev/null
+++ b/testinfra/test_refstack.py
@@ -0,0 +1,58 @@
+# Copyright 2020 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+import time
+import urllib3
+
+
+testinfra_hosts = ['refstack01.openstack.org:8000']
+
+test_result_json = {
+ "cpid": "9cddf99456964d7c90b98362e7175a12",
+ "duration_seconds": 10,
+ "results": [
+ {
+ "name": "tempest.api.compute.flavors.test_flavors."
+ "FlavorsV2TestJSON.test_list_flavors",
+ "uuid": "e36c0eaa-dff5-4082-ad1f-3f9a80aa3f59"
+ }
+ ]
+}
+
+
+def test_refstack_listening(host):
+ # Give it some time to come up
+ for i in range(300):
+ refstack_https = host.socket("tcp://0.0.0.0:443")
+ if refstack_https.is_listening:
+ break
+ time.sleep(1)
+ assert refstack_https.is_listening
+ refstack_http = host.socket("tcp://0.0.0.0:8000")
+ assert refstack_http.is_listening
+
+def test_refstack_container_running(host):
+ cmd = host.run("docker inspect refstack-docker_refstack-api_1")
+ out = json.loads(cmd.stdout)
+ assert out[0]["State"]["Status"] == "running"
+ assert out[0]["RestartCount"] == 0
+
+def test_result_submission(host):
+ url = testinfra_hosts[0] + "/v1/results/"
+ headers = {'Content-type': 'application/json'}
+ data = json.dumps(test_result_json)
+ http = urllib3.PoolManager()
+ resp = http.request('POST', url, body=data, headers=headers)
+ assert resp.status == 201
diff --git a/zuul.d/docker-images/refstack.yaml b/zuul.d/docker-images/refstack.yaml
new file mode 100644
index 0000000000..94feb59551
--- /dev/null
+++ b/zuul.d/docker-images/refstack.yaml
@@ -0,0 +1,25 @@
+# Refstack jobs
+- job:
+ name: system-config-build-image-refstack
+ description: Build a refstack image.
+ parent: system-config-build-image
+ vars: &refstack_vars
+ docker_images:
+ - context: docker/refstack
+ target: refstack
+ repository: opendevorg/refstack
+ # Duplicate in the run-refstack job
+ files: &refstack_files
+ - docker/refstack/.*
+- job:
+ name: system-config-upload-image-refstack
+ description: Build and upload a refstack image.
+ parent: system-config-upload-image
+ vars: *refstack_vars
+ files: *refstack_files
+- job:
+ name: system-config-promote-image-refstack
+ description: Promote a previously published refstack image to latest.
+ parent: system-config-promote-image
+ vars: *refstack_vars
+ files: *refstack_files
diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
index 4923dc1e3b..397a0c6dae 100644
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@@ -60,6 +60,11 @@
- name: opendev-buildset-registry
- name: system-config-build-image-gerrit-3.2
soft: true
+ - system-config-build-image-refstack
+ - system-config-run-refstack:
+ dependencies:
+ - name: system-config-build-image-refstack
+ soft: true
- system-config-run-zookeeper
- system-config-run-zuul
- system-config-run-zuul-preview
diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml
index 45413988b0..0787adeb82 100644
--- a/zuul.d/system-config-run.yaml
+++ b/zuul.d/system-config-run.yaml
@@ -889,3 +889,38 @@
'/var/log/acme.sh/': logs
'/etc/apache2/': logs
'/var/log/apache2/': logs
+
+- job:
+ name: system-config-run-refstack
+ parent: system-config-run
+ ansible-version: 2.9
+ description: |
+ Run the playbook for refstack server.
+ dependencies: opendev-buildset-registry
+ timeout: 3600
+ nodeset:
+ nodes:
+ - name: bridge.openstack.org
+ label: ubuntu-bionic
+ - name: refstack01.openstack.org
+ label: ubuntu-focal
+ host-vars:
+ refstack01.openstack.org:
+ host_copy_output:
+ '/var/lib/refstack/': logs
+ '/var/refstack/': logs
+ vars:
+ run_playbooks:
+ - playbooks/letsencrypt.yaml
+ - playbooks/service-refstack.yaml
+ container_command: docker
+ files:
+ - playbooks/bridge.yaml
+ - playbooks/group_vars/refstack.yaml
+ - playbooks/host_vars/gitea.*
+ - playbooks/zuul/templates/group_vars/refstack.yaml.j2
+ - playbooks/roles/refstack/
+ - playbooks/roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+ - testinfra/test_refstack.py
+ # If we rebuild the image, we want to run this job as well.
+ - docker/refstack/.*