From a4604ae0b3a9890befdda56ad45d2af961a109f0 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Fri, 31 Jan 2020 09:27:34 -0800 Subject: [PATCH] Deploy refstack with ansible docker This adds a dockerfile to build an opendevorg/refstack image as well as the jobs to build and publish it. Change-Id: Icade6c713fa9bf6ab508fd4d8d65debada2ddb30 --- docker/refstack/Dockerfile | 55 ++++++++ docker/refstack/entrypoint.sh | 4 + inventory/service/groups.yaml | 11 +- .../host_vars/refstack01.openstack.org.yaml | 4 + .../handlers/main.yaml | 5 + playbooks/roles/refstack/README.rst | 1 + playbooks/roles/refstack/files/robots.txt | 2 + playbooks/roles/refstack/handlers/main.yaml | 4 + playbooks/roles/refstack/tasks/main.yaml | 121 ++++++++++++++++++ .../roles/refstack/templates/config.json.j2 | 1 + .../refstack/templates/docker-compose.yaml.j2 | 24 ++++ .../roles/refstack/templates/refstack.conf.j2 | 17 +++ .../refstack/templates/refstack.vhost.j2 | 53 ++++++++ playbooks/service-refstack.yaml | 5 + playbooks/zuul/run-base.yaml | 2 + .../group_vars/refstack-docker.yaml.j2 | 5 + .../refstack01.openstack.org.yaml.j2 | 1 + testinfra/test_refstack.py | 58 +++++++++ zuul.d/docker-images/refstack.yaml | 25 ++++ zuul.d/project.yaml | 5 + zuul.d/system-config-run.yaml | 35 +++++ 21 files changed, 434 insertions(+), 4 deletions(-) create mode 100644 docker/refstack/Dockerfile create mode 100755 docker/refstack/entrypoint.sh create mode 100644 inventory/service/host_vars/refstack01.openstack.org.yaml create mode 100644 playbooks/roles/refstack/README.rst create mode 100644 playbooks/roles/refstack/files/robots.txt create mode 100644 playbooks/roles/refstack/handlers/main.yaml create mode 100644 playbooks/roles/refstack/tasks/main.yaml create mode 100644 playbooks/roles/refstack/templates/config.json.j2 create mode 100644 playbooks/roles/refstack/templates/docker-compose.yaml.j2 create mode 100644 playbooks/roles/refstack/templates/refstack.conf.j2 create mode 100644 playbooks/roles/refstack/templates/refstack.vhost.j2 create mode 100644 playbooks/service-refstack.yaml create mode 100644 playbooks/zuul/templates/group_vars/refstack-docker.yaml.j2 create mode 100644 playbooks/zuul/templates/host_vars/refstack01.openstack.org.yaml.j2 create mode 100644 testinfra/test_refstack.py create mode 100644 zuul.d/docker-images/refstack.yaml diff --git a/docker/refstack/Dockerfile b/docker/refstack/Dockerfile new file mode 100644 index 0000000000..4e189a29fa --- /dev/null +++ b/docker/refstack/Dockerfile @@ -0,0 +1,55 @@ +# Copyright (c) 2020 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM opendevorg/python-builder as builder + +RUN apt-get update \ + && apt-get -y install git apt-utils python3-dev dh-python libc-dev-bin \ + libc6-dev libexpat1-dev libpython3-dev libpython3-dev linux-libc-dev \ + apt-transport-https curl gnupg2 \ + && curl -sS https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \ + && echo "deb https://deb.nodesource.com/node_15.x bionic main" | tee /etc/apt/sources.list.d/nodesource.list \ + && curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \ + && echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -q --option "Dpkg::Options::=--force-confold" --assume-yes install nodejs yarn \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN git clone https://opendev.org/osf/refstack /tmp/src + +RUN assemble + +RUN cd /tmp/src && yarn install + +# Refstack's docs are built and then hosted by refstack UI +RUN python -m venv /tmp/venv \ + && /tmp/venv/bin/pip install beautifulsoup4 docutils \ + && /tmp/venv/bin/python /tmp/src/tools/convert-docs.py -o /tmp/src/refstack-ui/app/components/about/templates /tmp/src/doc/source/*.rst \ + && rm -rf /tmp/venv + +FROM opendevorg/python-base as refstack + +COPY --from=builder /output/ /output +COPY --from=builder /tmp/src/refstack-ui/app/ /refstack-ui/app +COPY ./entrypoint.sh /usr/bin/entrypoint +# TODO this should be fixed probably through proper js packaging +RUN rm /refstack-ui/app/assets/lib +COPY --from=builder /tmp/src/node_modules/@bower_components/ /refstack-ui/app/assets/lib +RUN /output/install-from-bindep \ + && rm -rf /output + +ENTRYPOINT ["/usr/bin/entrypoint"] +CMD ["pecan", "serve", "/usr/local/lib/python3.7/site-packages/refstack/api/config.py"] diff --git a/docker/refstack/entrypoint.sh b/docker/refstack/entrypoint.sh new file mode 100755 index 0000000000..a701d07ea8 --- /dev/null +++ b/docker/refstack/entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e +refstack-manage upgrade --revision head +$@ diff --git a/inventory/service/groups.yaml b/inventory/service/groups.yaml index b21282bf12..47efe54cb0 100644 --- a/inventory/service/groups.yaml +++ b/inventory/service/groups.yaml @@ -110,6 +110,7 @@ groups: - review-test.opendev.org - static[0-9]*.opendev.org - zuul[0-9]*.open*.org + - refstack[0-9]*.openstack.org logstash: - logstash[0-9]*.open*.org logstash-worker: @@ -165,7 +166,7 @@ groups: - paste[0-9]*.open*.org - pbx[0-9]*.opendev.org - planet[0-9]*.open*.org - - refstack*.open*.org + - refstack.openstack.org - status*.open*.org - storyboard-dev[0-9]*.opendev.org - storyboard[0-9]*.opendev.org @@ -198,7 +199,7 @@ groups: - paste[0-9]*.open*.org - pbx[0-9]*.opendev.org - planet[0-9]*.open*.org - - refstack*.open*.org + - refstack.openstack.org - status*.open*.org - storyboard[0-9]*.opendev.org - storyboard-dev[0-9]*.opendev.org @@ -209,7 +210,9 @@ groups: - wiki[0-9]*.openstack.org - wiki-dev[0-9]*.openstack.org refstack: - - refstack*.open*.org + - refstack[0-9]*.openstack.org + refstack-docker: + - refstack[0-9]*.openstack.org registry: - insecure-ci-registry[0-9]*.opendev.org review-dev: @@ -250,7 +253,7 @@ groups: - openstackid[0-9]*.openstack.org - paste[0-9]*.open*.org - planet[0-9]*.open*.org - - refstack*.open*.org + - refstack*.openstack.org - static[0-9]*.opendev.org - status*.open*.org - storyboard-dev[0-9]*.opendev.org diff --git a/inventory/service/host_vars/refstack01.openstack.org.yaml b/inventory/service/host_vars/refstack01.openstack.org.yaml new file mode 100644 index 0000000000..c4bbc4f8e1 --- /dev/null +++ b/inventory/service/host_vars/refstack01.openstack.org.yaml @@ -0,0 +1,4 @@ +letsencrypt_certs: + refstack01-openstack-org-main: + - refstack01.openstack.org + - refstack.openstack.org diff --git a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml index be04e2f520..abf40c848b 100644 --- a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml +++ b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml @@ -149,6 +149,11 @@ - name: letsencrypt updated review-test-opendev-org-main include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml +# refstack + +- name: letsencrypt updated refstack01-openstack-org-main + include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml + # Mirrors - name: letsencrypt updated mirror01-dfw-rax-main diff --git a/playbooks/roles/refstack/README.rst b/playbooks/roles/refstack/README.rst new file mode 100644 index 0000000000..cdffcb12bb --- /dev/null +++ b/playbooks/roles/refstack/README.rst @@ -0,0 +1 @@ +Install, configure, and run a refstack server. diff --git a/playbooks/roles/refstack/files/robots.txt b/playbooks/roles/refstack/files/robots.txt new file mode 100644 index 0000000000..1f53798bb4 --- /dev/null +++ b/playbooks/roles/refstack/files/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/playbooks/roles/refstack/handlers/main.yaml b/playbooks/roles/refstack/handlers/main.yaml new file mode 100644 index 0000000000..2ebc74c3c8 --- /dev/null +++ b/playbooks/roles/refstack/handlers/main.yaml @@ -0,0 +1,4 @@ +- name: refstack Reload apache2 + service: + name: apache2 + state: reloaded diff --git a/playbooks/roles/refstack/tasks/main.yaml b/playbooks/roles/refstack/tasks/main.yaml new file mode 100644 index 0000000000..420386623a --- /dev/null +++ b/playbooks/roles/refstack/tasks/main.yaml @@ -0,0 +1,121 @@ +- name: Ensure docker-compose directory exists + file: + state: directory + path: /etc/refstack-docker + mode: 0700 + +- name: Write docker-compose file + template: + src: docker-compose.yaml.j2 + dest: /etc/refstack-docker/docker-compose.yaml + mode: 0600 + +- name: Install apache2 + apt: + name: + - apache2 + - apache2-utils + state: present + +- name: Apache modules + apache2_module: + state: present + name: "{{ item }}" + loop: + - rewrite + - proxy + - proxy_http + - ssl + - headers + - proxy_wstunnel + +- name: Copy apache config + template: + src: refstack.vhost.j2 + dest: /etc/apache2/sites-enabled/000-default.conf + owner: root + group: root + mode: 0644 + notify: refstack Reload apache2 + +- name: Create refstack data storage area + file: + state: directory + path: /var/lib/refstack/data + owner: root + group: root + mode: 0755 + +- name: Create refstack www storage area + file: + state: directory + path: /var/lib/refstack/www + owner: root + group: root + mode: 0755 + +- name: Copy hound robots.txt + copy: + src: robots.txt + dest: /var/lib/refstack/www/robots.txt + +- name: Ensure refstack volume directory exists + file: + state: directory + path: "/var/refstack" + +- name: Write refstack.conf + template: + src: refstack.conf.j2 + dest: /var/refstack/refstack.conf + +- name: Write config.json + template: + src: config.json.j2 + dest: /var/refstack/config.json + +- name: Install distro packages + package: + name: + - docker-compose + state: present + +- name: Run docker-compose pull + shell: + cmd: docker-compose pull + chdir: /etc/refstack-docker/ + +- name: Run docker-compose up + shell: + cmd: docker-compose up -d --timeout 60 + chdir: /etc/refstack-docker/ + +- name: Run docker prune to cleanup unneeded images + shell: + cmd: docker image prune -f + +- name: Create db backup dest + file: + state: directory + path: /var/backups/refstack-mariadb + mode: 0700 + owner: root + group: root + +- name: Set up cron job to backup the database + cron: + name: refstack-db-backup + state: present + user: root + job: > + /usr/bin/docker-compose -f /etc/refstack-docker/docker-compose.yaml exec -T mariadb + bash -c '/usr/bin/mysqldump --opt --databases refstack --single-transaction -uroot -p"$MYSQL_ROOT_PASSWORD"' | + gzip -9 > /var/backups/refstack-mariadb/refstack-mariadb.sql.gz + minute: "42" + hour: "4" + +- name: Rotate db backups + include_role: + name: logrotate + vars: + logrotate_file_name: /var/backups/refstack-mariadb/refstack-mariadb.sql.gz diff --git a/playbooks/roles/refstack/templates/config.json.j2 b/playbooks/roles/refstack/templates/config.json.j2 new file mode 100644 index 0000000000..ac79e2c242 --- /dev/null +++ b/playbooks/roles/refstack/templates/config.json.j2 @@ -0,0 +1 @@ +{"refstackApiUrl": "{{ refstack_url }}/v1"} diff --git a/playbooks/roles/refstack/templates/docker-compose.yaml.j2 b/playbooks/roles/refstack/templates/docker-compose.yaml.j2 new file mode 100644 index 0000000000..3b6e5ba914 --- /dev/null +++ b/playbooks/roles/refstack/templates/docker-compose.yaml.j2 @@ -0,0 +1,24 @@ +# Version 2 is the latest that is supported by docker-compose in +# Ubuntu Xenial. +version: '2' + +services: + mariadb: + image: docker.io/library/mariadb:10.4 + network_mode: host + restart: always + environment: + MYSQL_ROOT_PASSWORD: "{{ refstack_root_db_password }}" + MYSQL_DATABASE: refstack + MYSQL_USER: "{{ refstack_db_username }}" + MYSQL_PASSWORD: "{{ refstack_db_password }}" + refstack-api: + depends_on: + - mariadb + image: opendevorg/refstack:latest + network_mode: host + restart: always + volumes: + - /var/refstack/refstack.conf:/etc/refstack.conf + - /var/refstack/config.json:/refstack-ui/app/config.json + - /var/lib/refstack/data:/var/run/data diff --git a/playbooks/roles/refstack/templates/refstack.conf.j2 b/playbooks/roles/refstack/templates/refstack.conf.j2 new file mode 100644 index 0000000000..7b7669079f --- /dev/null +++ b/playbooks/roles/refstack/templates/refstack.conf.j2 @@ -0,0 +1,17 @@ +[DEFAULT] +debug = true +verbose = true +ui_url = {{ refstack_url }} + +[api] +static_root = /refstack-ui/app +template_path = /refstack-ui/app +app_dev_mode = true +api_url = {{ refstack_url }} +enable_anonymous_upload = {{ enable_anonymous_upload | default(false) }} + +[database] +connection = "mysql+pymysql://{{ refstack_db_username}}:{{ refstack_db_password }}@localhost/refstack?charset=utf8" + +[osid] +openstack_openid_endpoint = {{ refstack_openid_endpoint }} diff --git a/playbooks/roles/refstack/templates/refstack.vhost.j2 b/playbooks/roles/refstack/templates/refstack.vhost.j2 new file mode 100644 index 0000000000..cdbbc24ab7 --- /dev/null +++ b/playbooks/roles/refstack/templates/refstack.vhost.j2 @@ -0,0 +1,53 @@ + + ServerName {{ inventory_hostname }} + ServerAdmin infra-root@openstack.org + + ErrorLog ${APACHE_LOG_DIR}/refstack-error.log + + LogLevel warn + + CustomLog ${APACHE_LOG_DIR}/refstack-access.log combined + + Redirect / https://refstack.openstack.org/ + + + + + ServerName {{ inventory_hostname }} + ServerAdmin webmaster@openstack.org + + RewriteCond %{HTTP_HOST} !^refstack\.openstack\.org [nocase] + RewriteRule ^/(.*) https://refstack.openstack.org/$1 [last,redirect=permanent] + + AllowEncodedSlashes On + + ErrorLog ${APACHE_LOG_DIR}/refstack-ssl-error.log + + LogLevel warn + + CustomLog ${APACHE_LOG_DIR}/refstack-ssl-access.log combined + + SSLEngine on + SSLProtocol All -SSLv2 -SSLv3 + # Note: this list should ensure ciphers that provide forward secrecy + SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP + SSLHonorCipherOrder on + + SSLCertificateFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.cer + SSLCertificateKeyFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.key + SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer + + ProxyPass / http://localhost:8000/ retry=0 + ProxyPassReverse / http://localhost:8000/ + + + ProxyPass ! + + + Require all granted + + Alias /robots.txt /var/lib/refstack/www/robots.txt + + + + diff --git a/playbooks/service-refstack.yaml b/playbooks/service-refstack.yaml new file mode 100644 index 0000000000..68a76d9650 --- /dev/null +++ b/playbooks/service-refstack.yaml @@ -0,0 +1,5 @@ +- hosts: "refstack-docker:!disabled" + name: "Configure refstack service in docker" + roles: + - install-docker + - refstack diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml index f660335124..50aa0e6603 100644 --- a/playbooks/zuul/run-base.yaml +++ b/playbooks/zuul/run-base.yaml @@ -62,6 +62,7 @@ - group_vars/meetpad.yaml - group_vars/jvb.yaml - group_vars/nodepool-launcher.yaml + - group_vars/refstack-docker.yaml - group_vars/registry.yaml - group_vars/review.yaml - group_vars/control-plane-clouds.yaml @@ -84,6 +85,7 @@ - host_vars/mirror-update01.opendev.org.yaml - host_vars/backup-test01.opendev.org.yaml - host_vars/backup-test02.opendev.org.yaml + - host_vars/refstack01.openstack.org.yaml - name: Display group membership command: ansible localhost -m debug -a 'var=groups' - name: Run base.yaml diff --git a/playbooks/zuul/templates/group_vars/refstack-docker.yaml.j2 b/playbooks/zuul/templates/group_vars/refstack-docker.yaml.j2 new file mode 100644 index 0000000000..2954b9d534 --- /dev/null +++ b/playbooks/zuul/templates/group_vars/refstack-docker.yaml.j2 @@ -0,0 +1,5 @@ +refstack_url: http://{{ ansible_fqdn }}:8000 +refstack_db_username: refstack +refstack_db_password: Jz4ooq9TL7nc3hX3 +refstack_root_db_password: KbgY3r9HYnEYpgRP +refstack_openid_endpoint: https://openstackid.org diff --git a/playbooks/zuul/templates/host_vars/refstack01.openstack.org.yaml.j2 b/playbooks/zuul/templates/host_vars/refstack01.openstack.org.yaml.j2 new file mode 100644 index 0000000000..e42664697f --- /dev/null +++ b/playbooks/zuul/templates/host_vars/refstack01.openstack.org.yaml.j2 @@ -0,0 +1 @@ +enable_anonymous_upload: true diff --git a/testinfra/test_refstack.py b/testinfra/test_refstack.py new file mode 100644 index 0000000000..b126f02d45 --- /dev/null +++ b/testinfra/test_refstack.py @@ -0,0 +1,58 @@ +# Copyright 2020 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import time +import urllib3 + + +testinfra_hosts = ['refstack01.openstack.org:8000'] + +test_result_json = { + "cpid": "9cddf99456964d7c90b98362e7175a12", + "duration_seconds": 10, + "results": [ + { + "name": "tempest.api.compute.flavors.test_flavors." + "FlavorsV2TestJSON.test_list_flavors", + "uuid": "e36c0eaa-dff5-4082-ad1f-3f9a80aa3f59" + } + ] +} + + +def test_refstack_listening(host): + # Give it some time to come up + for i in range(300): + refstack_https = host.socket("tcp://0.0.0.0:443") + if refstack_https.is_listening: + break + time.sleep(1) + assert refstack_https.is_listening + refstack_http = host.socket("tcp://0.0.0.0:8000") + assert refstack_http.is_listening + +def test_refstack_container_running(host): + cmd = host.run("docker inspect refstack-docker_refstack-api_1") + out = json.loads(cmd.stdout) + assert out[0]["State"]["Status"] == "running" + assert out[0]["RestartCount"] == 0 + +def test_result_submission(host): + url = testinfra_hosts[0] + "/v1/results/" + headers = {'Content-type': 'application/json'} + data = json.dumps(test_result_json) + http = urllib3.PoolManager() + resp = http.request('POST', url, body=data, headers=headers) + assert resp.status == 201 diff --git a/zuul.d/docker-images/refstack.yaml b/zuul.d/docker-images/refstack.yaml new file mode 100644 index 0000000000..94feb59551 --- /dev/null +++ b/zuul.d/docker-images/refstack.yaml @@ -0,0 +1,25 @@ +# Refstack jobs +- job: + name: system-config-build-image-refstack + description: Build a refstack image. + parent: system-config-build-image + vars: &refstack_vars + docker_images: + - context: docker/refstack + target: refstack + repository: opendevorg/refstack + # Duplicate in the run-refstack job + files: &refstack_files + - docker/refstack/.* +- job: + name: system-config-upload-image-refstack + description: Build and upload a refstack image. + parent: system-config-upload-image + vars: *refstack_vars + files: *refstack_files +- job: + name: system-config-promote-image-refstack + description: Promote a previously published refstack image to latest. + parent: system-config-promote-image + vars: *refstack_vars + files: *refstack_files diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 4923dc1e3b..397a0c6dae 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -60,6 +60,11 @@ - name: opendev-buildset-registry - name: system-config-build-image-gerrit-3.2 soft: true + - system-config-build-image-refstack + - system-config-run-refstack: + dependencies: + - name: system-config-build-image-refstack + soft: true - system-config-run-zookeeper - system-config-run-zuul - system-config-run-zuul-preview diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index 45413988b0..0787adeb82 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -889,3 +889,38 @@ '/var/log/acme.sh/': logs '/etc/apache2/': logs '/var/log/apache2/': logs + +- job: + name: system-config-run-refstack + parent: system-config-run + ansible-version: 2.9 + description: | + Run the playbook for refstack server. + dependencies: opendev-buildset-registry + timeout: 3600 + nodeset: + nodes: + - name: bridge.openstack.org + label: ubuntu-bionic + - name: refstack01.openstack.org + label: ubuntu-focal + host-vars: + refstack01.openstack.org: + host_copy_output: + '/var/lib/refstack/': logs + '/var/refstack/': logs + vars: + run_playbooks: + - playbooks/letsencrypt.yaml + - playbooks/service-refstack.yaml + container_command: docker + files: + - playbooks/bridge.yaml + - playbooks/group_vars/refstack.yaml + - playbooks/host_vars/gitea.* + - playbooks/zuul/templates/group_vars/refstack.yaml.j2 + - playbooks/roles/refstack/ + - playbooks/roles/letsencrypt-create-certs/handlers/restart_apache.yaml + - testinfra/test_refstack.py + # If we rebuild the image, we want to run this job as well. + - docker/refstack/.*