Deploy refstack with ansible docker

This adds a dockerfile to build an opendevorg/refstack image as well as
the jobs to build and publish it.

Change-Id: Icade6c713fa9bf6ab508fd4d8d65debada2ddb30

docker/refstack/Dockerfile

@@ -0,0 +1,55 @@
+# Copyright (c) 2020 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM opendevorg/python-builder as builder
+
+RUN apt-get update \
+  && apt-get -y install git apt-utils python3-dev dh-python libc-dev-bin \
+     libc6-dev libexpat1-dev libpython3-dev libpython3-dev linux-libc-dev \
+     apt-transport-https curl gnupg2 \
+  && curl -sS https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
+  && echo "deb https://deb.nodesource.com/node_15.x bionic main" | tee /etc/apt/sources.list.d/nodesource.list \
+  && curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
+  && echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
+  && apt-get update \
+  && DEBIAN_FRONTEND=noninteractive apt-get -q --option "Dpkg::Options::=--force-confold" --assume-yes install nodejs yarn \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN git clone https://opendev.org/osf/refstack /tmp/src
+
+RUN assemble
+
+RUN cd /tmp/src && yarn install
+
+# Refstack's docs are built and then hosted by refstack UI
+RUN python -m venv /tmp/venv \
+  && /tmp/venv/bin/pip install beautifulsoup4 docutils \
+  && /tmp/venv/bin/python /tmp/src/tools/convert-docs.py -o /tmp/src/refstack-ui/app/components/about/templates /tmp/src/doc/source/*.rst \
+  && rm -rf /tmp/venv
+
+FROM opendevorg/python-base as refstack
+
+COPY --from=builder /output/ /output
+COPY --from=builder /tmp/src/refstack-ui/app/ /refstack-ui/app
+COPY ./entrypoint.sh /usr/bin/entrypoint
+# TODO this should be fixed probably through proper js packaging
+RUN rm /refstack-ui/app/assets/lib
+COPY --from=builder /tmp/src/node_modules/@bower_components/ /refstack-ui/app/assets/lib
+RUN /output/install-from-bindep \
+  && rm -rf /output
+
+ENTRYPOINT ["/usr/bin/entrypoint"]
+CMD ["pecan", "serve", "/usr/local/lib/python3.7/site-packages/refstack/api/config.py"]

docker/refstack/entrypoint.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+refstack-manage upgrade --revision head
+$@

inventory/service/groups.yaml

@@ -110,6 +110,7 @@ groups:
     - review-test.opendev.org
     - static[0-9]*.opendev.org
     - zuul[0-9]*.open*.org
+    - refstack[0-9]*.openstack.org
   logstash:
     - logstash[0-9]*.open*.org
   logstash-worker:
@@ -165,7 +166,7 @@ groups:
     - paste[0-9]*.open*.org
     - pbx[0-9]*.opendev.org
     - planet[0-9]*.open*.org
-    - refstack*.open*.org
+    - refstack.openstack.org
     - status*.open*.org
     - storyboard-dev[0-9]*.opendev.org
     - storyboard[0-9]*.opendev.org
@@ -198,7 +199,7 @@ groups:
     - paste[0-9]*.open*.org
     - pbx[0-9]*.opendev.org
     - planet[0-9]*.open*.org
-    - refstack*.open*.org
+    - refstack.openstack.org
     - status*.open*.org
     - storyboard[0-9]*.opendev.org
     - storyboard-dev[0-9]*.opendev.org
@@ -209,7 +210,9 @@ groups:
     - wiki[0-9]*.openstack.org
     - wiki-dev[0-9]*.openstack.org
   refstack:
-    - refstack*.open*.org
+    - refstack[0-9]*.openstack.org
+  refstack-docker:
+    - refstack[0-9]*.openstack.org
   registry:
     - insecure-ci-registry[0-9]*.opendev.org
   review-dev:
@@ -250,7 +253,7 @@ groups:
     - openstackid[0-9]*.openstack.org
     - paste[0-9]*.open*.org
     - planet[0-9]*.open*.org
-    - refstack*.open*.org
+    - refstack*.openstack.org
     - static[0-9]*.opendev.org
     - status*.open*.org
     - storyboard-dev[0-9]*.opendev.org

inventory/service/host_vars/refstack01.openstack.org.yaml

@@ -0,0 +1,4 @@
+letsencrypt_certs:
+  refstack01-openstack-org-main:
+    - refstack01.openstack.org
+    - refstack.openstack.org

playbooks/roles/letsencrypt-create-certs/handlers/main.yaml

@@ -149,6 +149,11 @@
 - name: letsencrypt updated review-test-opendev-org-main
   include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
 
+# refstack
+
+- name: letsencrypt updated refstack01-openstack-org-main
+  include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+
 # Mirrors
 
 - name: letsencrypt updated mirror01-dfw-rax-main

playbooks/roles/refstack/README.rst

@@ -0,0 +1 @@
+Install, configure, and run a refstack server.

playbooks/roles/refstack/files/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

playbooks/roles/refstack/handlers/main.yaml

@@ -0,0 +1,4 @@
+- name: refstack Reload apache2
+  service:
+    name: apache2
+    state: reloaded

playbooks/roles/refstack/tasks/main.yaml

@@ -0,0 +1,121 @@
+- name: Ensure docker-compose directory exists
+  file:
+    state: directory
+    path: /etc/refstack-docker
+    mode: 0700
+
+- name: Write docker-compose file
+  template:
+    src: docker-compose.yaml.j2
+    dest: /etc/refstack-docker/docker-compose.yaml
+    mode: 0600
+
+- name: Install apache2
+  apt:
+    name:
+      - apache2
+      - apache2-utils
+    state: present
+
+- name: Apache modules
+  apache2_module:
+    state: present
+    name: "{{ item }}"
+  loop:
+    - rewrite
+    - proxy
+    - proxy_http
+    - ssl
+    - headers
+    - proxy_wstunnel
+
+- name: Copy apache config
+  template:
+    src: refstack.vhost.j2
+    dest: /etc/apache2/sites-enabled/000-default.conf
+    owner: root
+    group: root
+    mode: 0644
+  notify: refstack Reload apache2
+
+- name: Create refstack data storage area
+  file:
+    state: directory
+    path: /var/lib/refstack/data
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Create refstack www storage area
+  file:
+    state: directory
+    path: /var/lib/refstack/www
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Copy hound robots.txt
+  copy:
+    src: robots.txt
+    dest: /var/lib/refstack/www/robots.txt
+
+- name: Ensure refstack volume directory exists
+  file:
+    state: directory
+    path: "/var/refstack"
+
+- name: Write refstack.conf
+  template:
+    src: refstack.conf.j2
+    dest: /var/refstack/refstack.conf
+
+- name: Write config.json
+  template:
+    src: config.json.j2
+    dest: /var/refstack/config.json
+
+- name: Install distro packages
+  package:
+    name:
+      - docker-compose
+    state: present
+
+- name: Run docker-compose pull
+  shell:
+    cmd: docker-compose pull
+    chdir: /etc/refstack-docker/
+
+- name: Run docker-compose up
+  shell:
+    cmd: docker-compose up -d --timeout 60
+    chdir: /etc/refstack-docker/
+
+- name: Run docker prune to cleanup unneeded images
+  shell:
+    cmd: docker image prune -f
+
+- name: Create db backup dest
+  file:
+    state: directory
+    path: /var/backups/refstack-mariadb
+    mode: 0700
+    owner: root
+    group: root
+
+- name: Set up cron job to backup the database
+  cron:
+    name: refstack-db-backup
+    state: present
+    user: root
+    job: >
+      /usr/bin/docker-compose -f /etc/refstack-docker/docker-compose.yaml exec -T mariadb
+      bash -c '/usr/bin/mysqldump --opt --databases refstack --single-transaction -uroot -p"$MYSQL_ROOT_PASSWORD"' |
+      gzip -9 > /var/backups/refstack-mariadb/refstack-mariadb.sql.gz
+    minute: "42"
+    hour: "4"
+
+- name: Rotate db backups
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: /var/backups/refstack-mariadb/refstack-mariadb.sql.gz

playbooks/roles/refstack/templates/config.json.j2

@@ -0,0 +1 @@
+{"refstackApiUrl": "{{ refstack_url }}/v1"}

playbooks/roles/refstack/templates/docker-compose.yaml.j2

@@ -0,0 +1,24 @@
+# Version 2 is the latest that is supported by docker-compose in
+# Ubuntu Xenial.
+version: '2'
+
+services:
+  mariadb:
+    image: docker.io/library/mariadb:10.4
+    network_mode: host
+    restart: always
+    environment:
+      MYSQL_ROOT_PASSWORD: "{{ refstack_root_db_password }}"
+      MYSQL_DATABASE: refstack
+      MYSQL_USER: "{{ refstack_db_username }}"
+      MYSQL_PASSWORD: "{{ refstack_db_password }}"
+  refstack-api:
+    depends_on:
+      - mariadb
+    image: opendevorg/refstack:latest
+    network_mode: host
+    restart: always
+    volumes:
+      - /var/refstack/refstack.conf:/etc/refstack.conf
+      - /var/refstack/config.json:/refstack-ui/app/config.json
+      - /var/lib/refstack/data:/var/run/data

playbooks/roles/refstack/templates/refstack.conf.j2

@@ -0,0 +1,17 @@
+[DEFAULT]
+debug = true
+verbose = true
+ui_url = {{ refstack_url }}
+
+[api]
+static_root = /refstack-ui/app
+template_path = /refstack-ui/app
+app_dev_mode = true
+api_url = {{ refstack_url }}
+enable_anonymous_upload = {{ enable_anonymous_upload | default(false) }}
+
+[database]
+connection = "mysql+pymysql://{{ refstack_db_username}}:{{ refstack_db_password }}@localhost/refstack?charset=utf8"
+
+[osid]
+openstack_openid_endpoint = {{ refstack_openid_endpoint }}

playbooks/roles/refstack/templates/refstack.vhost.j2

@@ -0,0 +1,53 @@
+<VirtualHost *:80>
+  ServerName {{ inventory_hostname }}
+  ServerAdmin infra-root@openstack.org
+
+  ErrorLog ${APACHE_LOG_DIR}/refstack-error.log
+
+  LogLevel warn
+
+  CustomLog ${APACHE_LOG_DIR}/refstack-access.log combined
+
+  Redirect / https://refstack.openstack.org/
+
+</VirtualHost>
+
+<VirtualHost *:443>
+  ServerName {{ inventory_hostname }}
+  ServerAdmin webmaster@openstack.org
+
+  RewriteCond %{HTTP_HOST} !^refstack\.openstack\.org [nocase]
+  RewriteRule ^/(.*) https://refstack.openstack.org/$1 [last,redirect=permanent]
+
+  AllowEncodedSlashes On
+
+  ErrorLog ${APACHE_LOG_DIR}/refstack-ssl-error.log
+
+  LogLevel warn
+
+  CustomLog ${APACHE_LOG_DIR}/refstack-ssl-access.log combined
+
+  SSLEngine on
+  SSLProtocol All -SSLv2 -SSLv3
+  # Note: this list should ensure ciphers that provide forward secrecy
+  SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
+  SSLHonorCipherOrder on
+
+  SSLCertificateFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.cer
+  SSLCertificateKeyFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.key
+  SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer
+
+  ProxyPass  / http://localhost:8000/ retry=0
+  ProxyPassReverse / http://localhost:8000/
+
+  <Location "/robots.txt">
+    ProxyPass !
+  </Location>
+  <Directory "/var/lib/refstack/www">
+    Require all granted
+  </Directory>
+  Alias /robots.txt /var/lib/refstack/www/robots.txt
+
+</VirtualHost>
+
+

playbooks/service-refstack.yaml

@@ -0,0 +1,5 @@
+- hosts: "refstack-docker:!disabled"
+  name: "Configure refstack service in docker"
+  roles:
+    - install-docker
+    - refstack

playbooks/zuul/run-base.yaml

@@ -62,6 +62,7 @@
         - group_vars/meetpad.yaml
         - group_vars/jvb.yaml
         - group_vars/nodepool-launcher.yaml
+        - group_vars/refstack-docker.yaml
         - group_vars/registry.yaml
         - group_vars/review.yaml
         - group_vars/control-plane-clouds.yaml
@@ -84,6 +85,7 @@
         - host_vars/mirror-update01.opendev.org.yaml
         - host_vars/backup-test01.opendev.org.yaml
         - host_vars/backup-test02.opendev.org.yaml
+        - host_vars/refstack01.openstack.org.yaml
     - name: Display group membership
       command: ansible localhost -m debug -a 'var=groups'
     - name: Run base.yaml

playbooks/zuul/templates/group_vars/refstack-docker.yaml.j2

@@ -0,0 +1,5 @@
+refstack_url: http://{{ ansible_fqdn }}:8000
+refstack_db_username: refstack
+refstack_db_password: Jz4ooq9TL7nc3hX3
+refstack_root_db_password: KbgY3r9HYnEYpgRP
+refstack_openid_endpoint: https://openstackid.org

playbooks/zuul/templates/host_vars/refstack01.openstack.org.yaml.j2

@@ -0,0 +1 @@
+enable_anonymous_upload: true

testinfra/test_refstack.py

@@ -0,0 +1,58 @@
+# Copyright 2020 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+import time
+import urllib3
+
+
+testinfra_hosts = ['refstack01.openstack.org:8000']
+
+test_result_json = {
+    "cpid": "9cddf99456964d7c90b98362e7175a12",
+    "duration_seconds": 10,
+    "results": [
+        {
+            "name": "tempest.api.compute.flavors.test_flavors."
+                    "FlavorsV2TestJSON.test_list_flavors",
+            "uuid": "e36c0eaa-dff5-4082-ad1f-3f9a80aa3f59"
+        }
+    ]
+}
+
+
+def test_refstack_listening(host):
+    # Give it some time to come up
+    for i in range(300):
+        refstack_https = host.socket("tcp://0.0.0.0:443")
+        if refstack_https.is_listening:
+            break
+        time.sleep(1)
+    assert refstack_https.is_listening
+    refstack_http = host.socket("tcp://0.0.0.0:8000")
+    assert refstack_http.is_listening
+
+def test_refstack_container_running(host):
+    cmd = host.run("docker inspect refstack-docker_refstack-api_1")
+    out = json.loads(cmd.stdout)
+    assert out[0]["State"]["Status"] == "running"
+    assert out[0]["RestartCount"] == 0
+
+def test_result_submission(host):
+    url = testinfra_hosts[0] + "/v1/results/"
+    headers = {'Content-type': 'application/json'}
+    data = json.dumps(test_result_json)
+    http = urllib3.PoolManager()
+    resp = http.request('POST', url, body=data, headers=headers)
+    assert resp.status == 201

zuul.d/docker-images/refstack.yaml

@@ -0,0 +1,25 @@
+# Refstack jobs
+- job:
+    name: system-config-build-image-refstack
+    description: Build a refstack image.
+    parent: system-config-build-image
+    vars: &refstack_vars
+      docker_images:
+        - context: docker/refstack
+          target: refstack
+          repository: opendevorg/refstack
+    # Duplicate in the run-refstack job
+    files: &refstack_files
+      - docker/refstack/.*
+- job:
+    name: system-config-upload-image-refstack
+    description: Build and upload a refstack image.
+    parent: system-config-upload-image
+    vars: *refstack_vars
+    files: *refstack_files
+- job:
+    name: system-config-promote-image-refstack
+    description: Promote a previously published refstack image to latest.
+    parent: system-config-promote-image
+    vars: *refstack_vars
+    files: *refstack_files

zuul.d/project.yaml

@@ -60,6 +60,11 @@
               - name: opendev-buildset-registry
               - name: system-config-build-image-gerrit-3.2
                 soft: true
+        - system-config-build-image-refstack
+        - system-config-run-refstack:
+            dependencies:
+              - name: system-config-build-image-refstack
+                soft: true
         - system-config-run-zookeeper
         - system-config-run-zuul
         - system-config-run-zuul-preview

zuul.d/system-config-run.yaml

@@ -889,3 +889,38 @@
           '/var/log/acme.sh/': logs
           '/etc/apache2/': logs
           '/var/log/apache2/': logs
+
+- job:
+    name: system-config-run-refstack
+    parent: system-config-run
+    ansible-version: 2.9
+    description: |
+      Run the playbook for refstack server.
+    dependencies: opendev-buildset-registry
+    timeout: 3600
+    nodeset:
+      nodes:
+        - name: bridge.openstack.org
+          label: ubuntu-bionic
+        - name: refstack01.openstack.org
+          label: ubuntu-focal
+    host-vars:
+      refstack01.openstack.org:
+        host_copy_output:
+          '/var/lib/refstack/': logs
+          '/var/refstack/': logs
+    vars:
+      run_playbooks:
+        - playbooks/letsencrypt.yaml
+        - playbooks/service-refstack.yaml
+      container_command: docker
+    files:
+      - playbooks/bridge.yaml
+      - playbooks/group_vars/refstack.yaml
+      - playbooks/host_vars/gitea.*
+      - playbooks/zuul/templates/group_vars/refstack.yaml.j2
+      - playbooks/roles/refstack/
+      - playbooks/roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+      - testinfra/test_refstack.py
+      # If we rebuild the image, we want to run this job as well.
+      - docker/refstack/.*