From 12d435538583d7164b09c5c0c7ce3ffb04acdbc5 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Mon, 3 Oct 2022 14:32:23 -0700 Subject: [PATCH] Fork the maxking/docker-mailman images These images have a number of issues we've identified and worked around. The current iteration of this change is essentially identical to upstream but with a minor tweak to allow the latest mailman version, and adjusts the paths for hyperkitty and postorius URLs to match those in the upstream mailman-web codebase, but doesn't try to address the other items. However, we should consider moving our fixes from ansible into the docker images where possible and upstream those updates. Unfortunately upstream hasn't been super responsive so far hence this fork. For tracking purposes here are the issues/PRs we've already filed upstream: https://github.com/maxking/docker-mailman/pull/552 https://github.com/maxking/docker-mailman/issues/548 https://github.com/maxking/docker-mailman/issues/549 https://github.com/maxking/docker-mailman/issues/550 Change-Id: I3314037d46c2ef2086a06dea0321d9f8cdd35c73 --- docker/mailman/core/Dockerfile | 37 ++ docker/mailman/core/README.md | 6 + docker/mailman/core/docker-entrypoint.sh | 233 ++++++++++ docker/mailman/core/requirements.txt | 5 + docker/mailman/web/Dockerfile | 52 +++ docker/mailman/web/README.md | 6 + docker/mailman/web/docker-entrypoint.sh | 154 +++++++ docker/mailman/web/mailman-web/__init__.py | 0 docker/mailman/web/mailman-web/manage.py | 10 + docker/mailman/web/mailman-web/settings.py | 414 ++++++++++++++++++ docker/mailman/web/mailman-web/urls.py | 34 ++ docker/mailman/web/mailman-web/uwsgi.ini | 53 +++ docker/mailman/web/mailman-web/wsgi.py | 38 ++ docker/mailman/web/requirements.txt | 4 + .../mailman3/templates/docker-compose.yaml.j2 | 4 +- .../roles/mailman3/templates/mailman.vhost.j2 | 6 +- testinfra/test_lists_opendev_org.py | 12 +- tools/run-bashate.sh | 6 +- zuul.d/docker-images/mailman.yaml | 29 ++ zuul.d/project.yaml | 17 +- zuul.d/system-config-run.yaml | 3 +- 21 files changed, 1107 insertions(+), 16 deletions(-) create mode 100644 docker/mailman/core/Dockerfile create mode 100644 docker/mailman/core/README.md create mode 100755 docker/mailman/core/docker-entrypoint.sh create mode 100644 docker/mailman/core/requirements.txt create mode 100644 docker/mailman/web/Dockerfile create mode 100644 docker/mailman/web/README.md create mode 100755 docker/mailman/web/docker-entrypoint.sh create mode 100644 docker/mailman/web/mailman-web/__init__.py create mode 100755 docker/mailman/web/mailman-web/manage.py create mode 100644 docker/mailman/web/mailman-web/settings.py create mode 100644 docker/mailman/web/mailman-web/urls.py create mode 100644 docker/mailman/web/mailman-web/uwsgi.ini create mode 100755 docker/mailman/web/mailman-web/wsgi.py create mode 100644 docker/mailman/web/requirements.txt create mode 100644 zuul.d/docker-images/mailman.yaml diff --git a/docker/mailman/core/Dockerfile b/docker/mailman/core/Dockerfile new file mode 100644 index 0000000000..3c825a116c --- /dev/null +++ b/docker/mailman/core/Dockerfile @@ -0,0 +1,37 @@ +# syntax = docker/dockerfile:1.3 +# Use 3.15 for Core since it has Python 3.9 +FROM alpine:3.15 + +#Add startup script to container +COPY docker-entrypoint.sh /usr/local/bin/ + +# Add requirements file. +COPY requirements.txt /tmp/ + +#Install all required packages, add user for executing mailman and set execution rights for startup script +RUN --mount=type=cache,target=/root/.cache \ + apk update \ + && apk add --virtual build-deps gcc python3-dev musl-dev postgresql-dev \ + libffi-dev \ + # Mailman html to plaintext conversion uses lynx. + # psutil needs linux-headers to compile on musl c library. + && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers py-cryptography mariadb-connector-c lynx \ + && python3 -m pip install -U pip setuptools wheel \ + && python3 -m pip install psycopg2 \ + gunicorn==19.9.0 \ + pymysql \ + sqlalchemy \ + -r /tmp/requirements.txt \ + && apk del build-deps \ + && adduser -S mailman + +# Change the working directory. +WORKDIR /opt/mailman + +#Expose the ports for the api (8001) and lmtp (8024) +EXPOSE 8001 8024 + +ENV MAILMAN_CONFIG_FILE /etc/mailman.cfg + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["master", "--force"] diff --git a/docker/mailman/core/README.md b/docker/mailman/core/README.md new file mode 100644 index 0000000000..ce611f4f5f --- /dev/null +++ b/docker/mailman/core/README.md @@ -0,0 +1,6 @@ +This is forked from https://github.com/maxking/docker-mailman/tree/main/core +and you should refer to that repo for information. The upstream repo is MIT +licensed. + +The contents in this dir are based on commit +22b3b7d4024a00e9896837fa04883cbaeef38b20. diff --git a/docker/mailman/core/docker-entrypoint.sh b/docker/mailman/core/docker-entrypoint.sh new file mode 100755 index 0000000000..80aa6c79fa --- /dev/null +++ b/docker/mailman/core/docker-entrypoint.sh @@ -0,0 +1,233 @@ +#! /bin/bash +set -e + +function wait_for_postgres () { + # Check if the postgres database is up and accepting connections before + # moving forward. + # TODO: Use python3's psycopg2 module to do this in python3 instead of + # installing postgres-client in the image. + until psql $DATABASE_URL -c '\l'; do + >&2 echo "Postgres is unavailable - sleeping" + sleep 1 + done + >&2 echo "Postgres is up - continuing" +} + +function wait_for_mysql () { + # Check if MySQL is up and accepting connections. + readarray -d' ' -t ENDPOINT <<< $(python3 -c "from urllib.parse import urlparse; o = urlparse('$DATABASE_URL'); print('%s %s' % (o.hostname, o.port if o.port else '3306'));") + until mysqladmin ping --host ${ENDPOINT[0]} --port ${ENDPOINT[1]} --silent; do + >&2 echo "MySQL is unavailable - sleeping" + sleep 1 + done + >&2 echo "MySQL is up - continuing" +} + +# Empty the config file. +echo "# This file is autogenerated at container startup." > /etc/mailman.cfg + +# Check if $MM_HOSTNAME is set, if not, set it to the value returned by +# `hostname -i` command to set it to whatever IP address is assigned to the +# container. +if [[ ! -v MM_HOSTNAME ]]; then + export MM_HOSTNAME=`hostname -i` +fi + +# SMTP_HOST defaults to the gateway +if [[ ! -v SMTP_HOST ]]; then + export SMTP_HOST=$(/sbin/ip route | awk '/default/ { print $3 }') + echo "SMTP_HOST not specified, using the gateway ($SMTP_HOST) as default" +fi + +if [[ ! -v SMTP_PORT ]]; then + export SMTP_PORT=25 +fi + +# Check if REST port, username, and password are set, if not, set them +# to default values. +if [[ ! -v MAILMAN_REST_PORT ]]; then + export MAILMAN_REST_PORT='8001' +fi + +if [[ ! -v MAILMAN_REST_USER ]]; then + export MAILMAN_REST_USER='restadmin' +fi + +if [[ ! -v MAILMAN_REST_PASSWORD ]]; then + export MAILMAN_REST_PASSWORD='restpass' +fi + +function setup_database () { + if [[ ! -v DATABASE_URL ]] + then + echo "Environment variable DATABASE_URL should be defined..." + exit 1 + fi + + # Translate mysql:// urls to mysql+mysql:// backend: + if [[ "$DATABASE_URL" == mysql://* ]]; then + DATABASE_URL="mysql+pymysql://${DATABASE_URL:8}" + echo "Database URL was automatically rewritten to: $DATABASE_URL" + fi + + # If DATABASE_CLASS is not set, guess it for common databases: + if [ -z "$DATABASE_CLASS" ]; then + if [[ ("$DATABASE_URL" == mysql:*) || + ("$DATABASE_URL" == mysql+*) ]]; then + DATABASE_CLASS=mailman.database.mysql.MySQLDatabase + fi + if [[ ("$DATABASE_URL" == postgres:*) || + ("$DATABASE_URL" == postgres+*) ]]; then + DATABASE_CLASS=mailman.database.postgresql.PostgreSQLDatabase + fi + fi + + cat >> /etc/mailman.cfg <> /etc/mailman.cfg << EOF +[runner.retry] +sleep_time: 10s + +[webservice] +hostname: $MM_HOSTNAME +port: $MAILMAN_REST_PORT +admin_user: $MAILMAN_REST_USER +admin_pass: $MAILMAN_REST_PASSWORD +configuration: /etc/gunicorn.cfg + +EOF + +# Generate a basic gunicorn.cfg. +SITE_DIR=$(python3 -c 'import site; print(site.getsitepackages()[0])') +cp "${SITE_DIR}/mailman/config/gunicorn.cfg" /etc/gunicorn.cfg + +# Generate a basic configuration to use exim +cat > /tmp/exim-mailman.cfg < /etc/postfix-mailman.cfg << EOF +[postfix] +transport_file_type: regex +# While in regex mode, postmap_command is never used, a placeholder +# is added here so that it doesn't break anything. +postmap_command: true +EOF + +# Generate a basic configuration to use postfix. +cat > /tmp/postfix-mailman.cfg <> /etc/mailman.cfg +elif [ "$MTA" == "postfix" ] +then + echo "Using Postfix configuration" + cat /tmp/postfix-mailman.cfg >> /etc/mailman.cfg +else + echo "No MTA environment variable found, defaulting to Exim" + cat /tmp/exim-mailman.cfg >> /etc/mailman.cfg +fi + +rm -f /tmp/{postfix,exim}-mailman.cfg + +if [[ -e /opt/mailman/mailman-extra.cfg ]] +then + echo "Found configuration file at /opt/mailman/mailman-extra.cfg" + cat /opt/mailman/mailman-extra.cfg >> /etc/mailman.cfg +fi + +if [[ -e /opt/mailman/gunicorn-extra.cfg ]] +then + echo "Found [webserver] configuration file at /opt/mailman/gunicorn-extra.cfg" + cat /opt/mailman/gunicorn-extra.cfg > /etc/gunicorn.cfg +fi + +if [[ -v HYPERKITTY_API_KEY ]]; then + +echo "HYPERKITTY_API_KEY found, setting up HyperKitty archiver..." + +cat >> /etc/mailman.cfg << EOF +[archiver.hyperkitty] +class: mailman_hyperkitty.Archiver +enable: yes +configuration: /etc/mailman-hyperkitty.cfg + +EOF + +if [[ ! -v HYPERKITTY_URL ]]; then + echo "HYPERKITTY_URL not set, using the default value of http://mailman-web:8000/hyperkitty" + export HYPERKITTY_URL="http://mailman-web:8000/hyperkitty/" +fi + +# Generate a basic mailman-hyperkitty.cfg. +cat > /etc/mailman-hyperkitty.cfg <&2 echo "Postgres is unavailable - sleeping" + sleep 1 + done + >&2 echo "Postgres is up - continuing" +} + +function wait_for_mysql () { + # Check if MySQL is up and accepting connections. + readarray -d' ' -t ENDPOINT <<< $(python3 -c "from urllib.parse import urlparse; o = urlparse('$DATABASE_URL'); print('%s %s' % (o.hostname, o.port if o.port else '3306'));") + until mysqladmin ping --host ${ENDPOINT[0]} --port ${ENDPOINT[1]} --silent; do + >&2 echo "MySQL is unavailable - sleeping" + sleep 1 + done + >&2 echo "MySQL is up - continuing" +} + +function check_or_create () { + # Check if the path exists, if not, create the directory. + if [[ ! -e dir ]]; then + echo "$1 does not exist, creating ..." + mkdir "$1" + fi +} + +# function postgres_ready(){ +# python << END +# import sys +# import psycopg2 +# try: +# conn = psycopg2.connect(dbname="$POSTGRES_DB", user="$POSTGRES_USER", password="$POSTGRES_PASSWORD", host="postgres") +# except psycopg2.OperationalError: +# sys.exit(-1) +# sys.exit(0) +# END +# } + +# SMTP_HOST defaults to the gateway +if [[ ! -v SMTP_HOST ]]; then + export SMTP_HOST=$(/sbin/ip route | awk '/default/ { print $3 }') +fi + +# Check if $SECRET_KEY is defined, if not, bail out. +if [[ ! -v SECRET_KEY ]]; then + echo "SECRET_KEY is not defined. Aborting." + exit 1 +fi + +# Check if $DATABASE_URL is defined, if not, use a standard sqlite database. +# +# If the $DATABASE_URL is defined and is postgres, check if it is available +# yet. Do not start the container before the postgresql boots up. +# +# If the $DATABASE_URL is defined and is mysql, check if the database is +# available before the container boots up. +# +# TODO: Check the database type and detect if it is up based on that. For now, +# assume that postgres is being used if DATABASE_URL is defined. + +if [[ ! -v DATABASE_URL ]]; then + echo "DATABASE_URL is not defined. Using sqlite database..." + export DATABASE_URL=sqlite://mailmanweb.db + export DATABASE_TYPE='sqlite' +fi + +if [[ "$DATABASE_TYPE" = 'postgres' ]] +then + wait_for_postgres +elif [[ "$DATABASE_TYPE" = 'mysql' ]] +then + wait_for_mysql +fi + +# Check if we are in the correct directory before running commands. +if [[ ! $(pwd) == '/opt/mailman-web' ]]; then + echo "Running in the wrong directory...switching to /opt/mailman-web" + cd /opt/mailman-web +fi + +# Check if the logs directory is setup. +if [[ ! -e /opt/mailman-web-data/logs/mailmanweb.log ]]; then + echo "Creating log file for mailman web" + mkdir -p /opt/mailman-web-data/logs/ + touch /opt/mailman-web-data/logs/mailmanweb.log +fi + +if [[ ! -e /opt/mailman-web-data/logs/uwsgi.log ]]; then + echo "Creating log file for uwsgi.." + touch /opt/mailman-web-data/logs/uwsgi.log +fi + +# Check if the settings_local.py file exists, if yes, copy it too. +if [[ -e /opt/mailman-web-data/settings_local.py ]]; then + echo "Copying settings_local.py ..." + cp /opt/mailman-web-data/settings_local.py /opt/mailman-web/settings_local.py + chown mailman:mailman /opt/mailman-web/settings_local.py +else + echo "settings_local.py not found, it is highly recommended that you provide one" + echo "Using default configuration to run." +fi + +# Collect static for the django installation. +python3 manage.py collectstatic --noinput --clear --verbosity 0 + + +# Compile all the installed po files to mo. +SITE_DIR=$(python3 -c 'import site; print(site.getsitepackages()[0])') +echo "Compiling locale files in $SITE_DIR" +cd $SITE_DIR && /opt/mailman-web/manage.py compilemessages && cd - + +# Compress static files. +python3 manage.py compress --force + + +# Migrate all the data to the database if this is a new installation, otherwise +# this command will upgrade the database. +python3 manage.py migrate + +# If MAILMAN_ADMIN_USER and MAILMAN_ADMIN_EMAIL is defined create a new +# superuser for Django. There is no password setup so it can't login yet unless +# the password is reset. +if [[ -v MAILMAN_ADMIN_USER ]] && [[ -v MAILMAN_ADMIN_EMAIL ]]; +then + echo "Creating admin user $MAILMAN_ADMIN_USER ..." + python3 manage.py createsuperuser --noinput --username "$MAILMAN_ADMIN_USER"\ + --email "$MAILMAN_ADMIN_EMAIL" 2> /dev/null || \ + echo "Superuser $MAILMAN_ADMIN_USER already exists" +fi + +# If SERVE_FROM_DOMAIN is defined then rename the default `example.com` +# domain to the defined domain. +if [[ -v SERVE_FROM_DOMAIN ]]; +then + echo "Setting $SERVE_FROM_DOMAIN as the default domain ..." + python3 manage.py shell -c \ + "from django.contrib.sites.models import Site; Site.objects.filter(domain='example.com').update(domain='$SERVE_FROM_DOMAIN', name='$SERVE_FROM_DOMAIN')" +fi + +# Create a mailman user with the specific UID and GID and do not create home +# directory for it. Also chown the logs directory to write the files. +chown mailman:mailman /opt/mailman-web-data -R + +[[ -v DISKCACHE_PATH ]] && chown mailman:mailman "${DISKCACHE_PATH}" -R + +exec $@ diff --git a/docker/mailman/web/mailman-web/__init__.py b/docker/mailman/web/mailman-web/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/mailman/web/mailman-web/manage.py b/docker/mailman/web/mailman-web/manage.py new file mode 100755 index 0000000000..4eb6f348a4 --- /dev/null +++ b/docker/mailman/web/mailman-web/manage.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") + + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/docker/mailman/web/mailman-web/settings.py b/docker/mailman/web/mailman-web/settings.py new file mode 100644 index 0000000000..6c2d0d0d90 --- /dev/null +++ b/docker/mailman/web/mailman-web/settings.py @@ -0,0 +1,414 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. +# +# This file is part of Mailman Suite. +# +# Mailman Suite is free sofware: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# Mailman Suite is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. + +# You should have received a copy of the GNU General Public License along +# with Mailman Suite. If not, see . +""" +Django Settings for Mailman Suite (hyperkitty + postorius) + +For more information on this file, see +https://docs.djangoproject.com/en/1.8/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/1.8/ref/settings/ +""" + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +import os +import dj_database_url +import sys +from socket import gethostbyname + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.environ.get('SECRET_KEY') + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = False + +ADMINS = ( + ('Mailman Suite Admin', 'root@localhost'), +) + +SITE_ID = 1 + +# Hosts/domain names that are valid for this site; required if DEBUG is False +# See https://docs.djangoproject.com/en/3.1/ref/settings/#allowed-hosts +ALLOWED_HOSTS = [ + "localhost", # Archiving API from Mailman, keep it. + # "lists.your-domain.org", + # Add here all production URLs you may have. + "mailman-web", + gethostbyname("mailman-web"), + os.environ.get('SERVE_FROM_DOMAIN'), + os.environ.get('DJANGO_ALLOWED_HOSTS'), +] + +# Mailman API credentials +MAILMAN_REST_API_URL = os.environ.get('MAILMAN_REST_URL', 'http://mailman-core:8001') +MAILMAN_REST_API_USER = os.environ.get('MAILMAN_REST_USER', 'restadmin') +MAILMAN_REST_API_PASS = os.environ.get('MAILMAN_REST_PASSWORD', 'restpass') +MAILMAN_ARCHIVER_KEY = os.environ.get('HYPERKITTY_API_KEY') +MAILMAN_ARCHIVER_FROM = (os.environ.get('MAILMAN_HOST_IP', gethostbyname(os.environ.get('MAILMAN_HOSTNAME', 'mailman-core'))),) + +# Application definition + +INSTALLED_APPS = [] +DEFAULT_APPS = [ + 'hyperkitty', + 'postorius', + 'django_mailman3', + # Uncomment the next line to enable the admin: + 'django.contrib.admin', + # Uncomment the next line to enable admin documentation: + # 'django.contrib.admindocs', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'rest_framework', + 'django_gravatar', + 'compressor', + 'haystack', + 'django_extensions', + 'django_q', + 'allauth', + 'allauth.account', + 'allauth.socialaccount', +] + +MAILMAN_WEB_SOCIAL_AUTH = [ + 'django_mailman3.lib.auth.fedora', + 'allauth.socialaccount.providers.openid', + 'allauth.socialaccount.providers.github', + 'allauth.socialaccount.providers.gitlab', + 'allauth.socialaccount.providers.google', +] + +MIDDLEWARE = ( + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.middleware.locale.LocaleMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.middleware.security.SecurityMiddleware', + 'django_mailman3.middleware.TimezoneMiddleware', + 'postorius.middleware.PostoriusMiddleware', +) + +ROOT_URLCONF = 'urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.i18n', + 'django.template.context_processors.media', + 'django.template.context_processors.static', + 'django.template.context_processors.tz', + 'django.template.context_processors.csrf', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + 'django_mailman3.context_processors.common', + 'hyperkitty.context_processors.common', + 'postorius.context_processors.postorius', + ], + }, + }, +] + +WSGI_APPLICATION = 'wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/1.8/ref/settings/#databases +# dj_database_url uses $DATABASE_URL environment variable to create a +# django-style-config-dict. +# https://github.com/kennethreitz/dj-database-url +DATABASES = { + 'default': dj_database_url.config(conn_max_age=600) +} + +# If you're behind a proxy, use the X-Forwarded-Host header +# See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host +USE_X_FORWARDED_HOST = True + + +# Password validation +# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + +# Internationalization +# https://docs.djangoproject.com/en/1.8/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + +STATIC_ROOT = '/opt/mailman-web-data/static' + +STATIC_URL = '/static/' + +# Additional locations of static files + + +# List of finder classes that know how to find static files in +# various locations. +STATICFILES_FINDERS = ( + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + 'compressor.finders.CompressorFinder', +) + + +SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer' + +LOGIN_URL = 'account_login' +LOGIN_REDIRECT_URL = 'list_index' +LOGOUT_URL = 'account_logout' + + +# Use SERVE_FROM_DOMAIN as the default domain in the email. +hostname = os.environ.get('SERVE_FROM_DOMAIN', 'localhost.local') +DEFAULT_FROM_EMAIL = 'postorius@{}'.format(hostname) +SERVER_EMAIL = 'root@{}'.format(hostname) + +# Change this when you have a real email backend +EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' +EMAIL_HOST = os.environ.get('SMTP_HOST', '') +EMAIL_PORT = os.environ.get('SMTP_PORT', 25) +EMAIL_HOST_USER = os.environ.get('SMTP_HOST_USER', '') +EMAIL_HOST_PASSWORD = os.environ.get('SMTP_HOST_PASSWORD', '') +EMAIL_USE_TLS = os.environ.get('SMTP_USE_TLS', False) +EMAIL_USE_SSL = os.environ.get('SMTP_USE_SSL', False) + +# Compatibility with Bootstrap 3 +from django.contrib.messages import constants as messages # flake8: noqa +MESSAGE_TAGS = { + messages.ERROR: 'danger' +} + + +# +# Social auth +# +AUTHENTICATION_BACKENDS = ( + 'django.contrib.auth.backends.ModelBackend', + 'allauth.account.auth_backends.AuthenticationBackend', +) + +# Django Allauth +ACCOUNT_AUTHENTICATION_METHOD = "username_email" +ACCOUNT_EMAIL_REQUIRED = True +ACCOUNT_EMAIL_VERIFICATION = "mandatory" +# You probably want https in production, but this is a dev setup file +ACCOUNT_DEFAULT_HTTP_PROTOCOL = "https" +ACCOUNT_UNIQUE_EMAIL = True + +SOCIALACCOUNT_PROVIDERS = { + 'openid': { + 'SERVERS': [ + dict(id='yahoo', + name='Yahoo', + openid_url='http://me.yahoo.com'), + ], + }, + 'google': { + 'SCOPE': ['profile', 'email'], + 'AUTH_PARAMS': {'access_type': 'online'}, + }, + 'facebook': { + 'METHOD': 'oauth2', + 'SCOPE': ['email'], + 'FIELDS': [ + 'email', + 'name', + 'first_name', + 'last_name', + 'locale', + 'timezone', + ], + 'VERSION': 'v2.4', + }, +} + + +# django-compressor +# https://pypi.python.org/pypi/django_compressor +# +COMPRESS_PRECOMPILERS = ( + ('text/less', 'lessc {infile} {outfile}'), + ('text/x-scss', 'sassc -t compressed {infile} {outfile}'), + ('text/x-sass', 'sassc -t compressed {infile} {outfile}'), +) + +# On a production setup, setting COMPRESS_OFFLINE to True will bring a +# significant performance improvement, as CSS files will not need to be +# recompiled on each requests. It means running an additional "compress" +# management command after each code upgrade. +# http://django-compressor.readthedocs.io/en/latest/usage/#offline-compression +# COMPRESS_OFFLINE = True + +# +# Full-text search engine +# +HAYSTACK_CONNECTIONS = { + 'default': { + 'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine', + 'PATH': "/opt/mailman-web-data/fulltext_index", + # You can also use the Xapian engine, it's faster and more accurate, + # but requires another library. + # http://django-haystack.readthedocs.io/en/v2.4.1/installing_search_engines.html#xapian + # Example configuration for Xapian: + #'ENGINE': 'xapian_backend.XapianEngine' + }, +} + +import sys +# A sample logging configuration. The only tangible logging +# performed by this configuration is to send an email to +# the site admins on every HTTP 500 error when DEBUG=False. +# See http://docs.djangoproject.com/en/dev/topics/logging for +# more details on how to customize your logging configuration. +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse' + } + }, + 'handlers': { + 'mail_admins': { + 'level': 'ERROR', + 'filters': ['require_debug_false'], + 'class': 'django.utils.log.AdminEmailHandler' + }, + 'file':{ + 'level': 'INFO', + 'class': 'logging.handlers.RotatingFileHandler', + #'class': 'logging.handlers.WatchedFileHandler', + 'filename': os.environ.get('DJANGO_LOG_URL','/opt/mailman-web-data/logs/mailmanweb.log'), + 'formatter': 'verbose', + }, + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'simple', + 'level': 'INFO', + 'stream': sys.stdout, + }, + }, + 'loggers': { + 'django.request': { + 'handlers': ['mail_admins', 'file'], + 'level': 'INFO', + 'propagate': True, + }, + 'django': { + 'handlers': ['file'], + 'level': 'INFO', + 'propagate': True, + }, + 'hyperkitty': { + 'handlers': ['file'], + 'level': 'INFO', + 'propagate': True, + }, + 'postorius': { + 'handlers': ['file'], + 'level': 'INFO', + 'propagate': True + }, + }, + 'formatters': { + 'verbose': { + 'format': '%(levelname)s %(asctime)s %(process)d %(name)s %(message)s' + }, + 'simple': { + 'format': '%(levelname)s %(message)s' + }, + }, +} + + +if os.environ.get('LOG_TO_CONSOLE') == 'yes': + LOGGING['loggers']['django']['handlers'].append('console') + LOGGING['loggers']['django.request']['handlers'].append('console') + +# HyperKitty-specific +# +# Only display mailing-lists from the same virtual host as the webserver +FILTER_VHOST = False + + +Q_CLUSTER = { + 'timeout': 300, + 'retry': 300, + 'save_limit': 100, + 'orm': 'default', +} + +POSTORIUS_TEMPLATE_BASE_URL = os.environ.get('POSTORIUS_TEMPLATE_BASE_URL', 'http://mailman-web:8000') + +DISKCACHE_PATH = os.environ.get('DISKCACHE_PATH', '/opt/mailman-web-data/diskcache') +DISKCACHE_SIZE = os.environ.get('DISKCACHE_SIZE', 2 ** 30) # 1 gigabyte + +CACHES = { + 'default': { + 'BACKEND': 'diskcache.DjangoCache', + 'LOCATION': DISKCACHE_PATH, + 'OPTIONS': { + 'size_limit': DISKCACHE_SIZE, + }, + }, +} + +try: + from settings_local import * +except ImportError: + pass + +# Compatibility for older installs that override INSTALLED_APPS +if not INSTALLED_APPS: + INSTALLED_APPS = DEFAULT_APPS + MAILMAN_WEB_SOCIAL_AUTH diff --git a/docker/mailman/web/mailman-web/urls.py b/docker/mailman/web/mailman-web/urls.py new file mode 100644 index 0000000000..aa12b25f73 --- /dev/null +++ b/docker/mailman/web/mailman-web/urls.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. +# +# This file is part of Postorius. +# +# Postorius is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# Postorius is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Postorius. If not, see . + +from django.conf.urls import include +from django.contrib import admin +from django.urls import path, reverse_lazy +from django.views.generic import RedirectView + +urlpatterns = [ + path(r'', RedirectView.as_view( + url=reverse_lazy('list_index'), + permanent=True)), + path(r'mailman3/', include('postorius.urls')), + path(r'archives/', include('hyperkitty.urls')), + path(r'', include('django_mailman3.urls')), + path(r'accounts/', include('allauth.urls')), + # Django admin + path(r'admin/', admin.site.urls), +] diff --git a/docker/mailman/web/mailman-web/uwsgi.ini b/docker/mailman/web/mailman-web/uwsgi.ini new file mode 100644 index 0000000000..8e917a34f5 --- /dev/null +++ b/docker/mailman/web/mailman-web/uwsgi.ini @@ -0,0 +1,53 @@ +[uwsgi] +# Port on which uwsgi will be listening. +uwsgi-socket = 0.0.0.0:8080 +http-socket = 0.0.0.0:8000 + +#Enable threading for python +enable-threads = true + +# Setting uwsgi buffer size to what Apache2 supports. +buffer-size = 8190 + +# Move to the directory wher the django files are. +chdir = /opt/mailman-web + +# Use the wsgi file provided with the django project. +wsgi-file = wsgi.py + +# Setup default number of processes and threads per process. +master = true +processes = 2 +threads = 2 + +# Drop privielges and don't run as root. +uid = mailman +gid = mailman + +# Setup the django_q related worker processes. +attach-daemon = ./manage.py qcluster + +# Setup hyperkitty's cron jobs. +# 'minutely' jobs are run hourly for perf reasons. +# See https://github.com/maxking/docker-mailman/issues/327 +unique-cron = 0 -1 -1 -1 -1 ./manage.py runjobs minutely +unique-cron = -15 -1 -1 -1 -1 ./manage.py runjobs quarter_hourly +unique-cron = 0 -1 -1 -1 -1 ./manage.py runjobs hourly +unique-cron = 0 0 -1 -1 -1 ./manage.py runjobs daily +unique-cron = 0 0 1 -1 -1 ./manage.py runjobs monthly +unique-cron = 0 0 -1 -1 0 ./manage.py runjobs weekly +unique-cron = 0 0 1 1 -1 ./manage.py runjobs yearly + +# Setup the request log. +req-logger = file:/opt/mailman-web-data/logs/uwsgi.log + +# Log cron seperately. +logger = cron file:/opt/mailman-web-data/logs/uwsgi-cron.log +log-route = cron uwsgi-cron + +# Log qcluster commands seperately. +logger = qcluster file:/opt/mailman-web-data/logs/uwsgi-qcluster.log +log-route = qcluster uwsgi-daemons + +# Last log and it logs the rest of the stuff. +logger = file:/opt/mailman-web-data/logs/uwsgi-error.log diff --git a/docker/mailman/web/mailman-web/wsgi.py b/docker/mailman/web/mailman-web/wsgi.py new file mode 100755 index 0000000000..8e59cd8545 --- /dev/null +++ b/docker/mailman/web/mailman-web/wsgi.py @@ -0,0 +1,38 @@ +""" +WSGI config for HyperKitty project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/{{ docs_version }}/howto/deployment/wsgi/ +""" + +import os + +# import sys +# import site + +# For some unknown reason, sometimes mod_wsgi fails to set the python paths to +# the virtualenv, with the 'python-path' option. You can do it here too. +# +# # Remember original sys.path. +# prev_sys_path = list(sys.path) +# # Add here, for the settings module +# site.addsitedir(os.path.abspath(os.path.dirname(__file__))) +# # Add the virtualenv +# venv = os.path.join(os.path.abspath(os.path.dirname(__file__)), +# '..', 'lib', 'python2.6', 'site-packages') +# site.addsitedir(venv) +# # Reorder sys.path so new directories at the front. +# new_sys_path = [] +# for item in list(sys.path): +# if item not in prev_sys_path: +# new_sys_path.append(item) +# sys.path.remove(item) +# sys.path[:0] = new_sys_path + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") + +application = get_wsgi_application() diff --git a/docker/mailman/web/requirements.txt b/docker/mailman/web/requirements.txt new file mode 100644 index 0000000000..20dc945d92 --- /dev/null +++ b/docker/mailman/web/requirements.txt @@ -0,0 +1,4 @@ +mailmanclient==3.3.4 +postorius==1.3.7 +hyperkitty==1.3.6 +django-mailman3==1.3.8 \ No newline at end of file diff --git a/playbooks/roles/mailman3/templates/docker-compose.yaml.j2 b/playbooks/roles/mailman3/templates/docker-compose.yaml.j2 index a53ad65412..b0cf4c73b0 100644 --- a/playbooks/roles/mailman3/templates/docker-compose.yaml.j2 +++ b/playbooks/roles/mailman3/templates/docker-compose.yaml.j2 @@ -4,7 +4,7 @@ version: '2' services: mailman-core: - image: docker.io/maxking/mailman-core:0.4 + image: docker.io/opendevorg/mailman-core:latest restart: always container_name: mailman-core volumes: @@ -27,7 +27,7 @@ services: #user: mailman mailman-web: - image: docker.io/maxking/mailman-web:0.4 + image: docker.io/opendevorg/mailman-web:latest restart: always container_name: mailman-web depends_on: diff --git a/playbooks/roles/mailman3/templates/mailman.vhost.j2 b/playbooks/roles/mailman3/templates/mailman.vhost.j2 index 811b61c87c..e689af2f1a 100644 --- a/playbooks/roles/mailman3/templates/mailman.vhost.j2 +++ b/playbooks/roles/mailman3/templates/mailman.vhost.j2 @@ -39,7 +39,7 @@ SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer Alias /static /var/lib/mailman/web-data/static - Alias /favicon.ico /var/lib/mailman/web-data/static/hyperkitty/img/favicon.ico + Alias /favicon.ico /var/lib/mailman/web-data/static/archives/img/favicon.ico Require local @@ -47,8 +47,8 @@ RewriteEngine On RewriteRule "/pipermail/(.*)" "/var/lib/mailman/web-data/mm2archives/%{HTTP_HOST}/public/$1" - RewriteRule "/cgi-bin/mailman/listinfo/(.*)" "https://%{HTTP_HOST}/postorius/lists/$1.%{HTTP_HOST}/" - RewriteRule "/cgi-bin/mailman/listinfo" "https://%{HTTP_HOST}/postorius/lists/" + RewriteRule "/cgi-bin/mailman/listinfo/(.*)" "https://%{HTTP_HOST}/mailman3/lists/$1.%{HTTP_HOST}/" + RewriteRule "/cgi-bin/mailman/listinfo" "https://%{HTTP_HOST}/mailman3/lists/" ProxyPassMatch ^/static/ ! ProxyPass "/" "uwsgi://localhost:8080/" diff --git a/testinfra/test_lists_opendev_org.py b/testinfra/test_lists_opendev_org.py index 5b1ff89fc9..0dcc32834a 100644 --- a/testinfra/test_lists_opendev_org.py +++ b/testinfra/test_lists_opendev_org.py @@ -39,22 +39,22 @@ def test_apache2_listening(host): def test_mailman3_screenshots(host): shots = ( ("https://lists.opendev.org:443", None, "mm3-opendev-main.png"), - ("https://lists.opendev.org:443/hyperkitty/", + ("https://lists.opendev.org:443/archives/", None, "mm3-opendev-archives.png"), ("https://lists.opendev.org:443" - "/hyperkitty/list/service-discuss@lists.opendev.org/", + "/archives/list/service-discuss@lists.opendev.org/", None, "mm3-opendev-list.png"), ("https://lists.opendev.org:443" - "/accounts/login/?next=/postorius/lists/", + "/accounts/login/?next=/mailman3/lists/", None, "mm3-opendev-login.png"), ("https://lists.openstack.org:443", None, "mm3-openstack-main.png"), - ("https://lists.openstack.org:443/hyperkitty/", + ("https://lists.openstack.org:443/archives/", None, "mm3-openstack-archives.png"), ("https://lists.openstack.org:443" - "/hyperkitty/list/openstack-discuss@lists.openstack.org/", + "/archives/list/openstack-discuss@lists.openstack.org/", None, "mm3-openstack-list.png"), ("https://lists.openstack.org:443" - "/accounts/login/?next=/postorius/lists/", + "/accounts/login/?next=/mailman3/lists/", None, "mm3-openstack-login.png"), ) diff --git a/tools/run-bashate.sh b/tools/run-bashate.sh index e57e734c82..b38c4e10e6 100755 --- a/tools/run-bashate.sh +++ b/tools/run-bashate.sh @@ -1,6 +1,8 @@ #!/bin/bash ROOT=$(readlink -fn $(dirname $0)/.. ) -find $ROOT -type f -not -wholename \*.tox/\* -and \( -name \*.sh \ - -or -name \*rc -or -name functions\* \) -print0 \ +find $ROOT -type f -not -wholename \*.tox/\* -and \ + \( -name \*.sh -or -name \*rc -or -name functions\* \) -and \ + -not -path \*/docker/mailman/\* \ + -print0 \ | xargs -0 bashate -i E006,E010 -v diff --git a/zuul.d/docker-images/mailman.yaml b/zuul.d/docker-images/mailman.yaml new file mode 100644 index 0000000000..783d338cb6 --- /dev/null +++ b/zuul.d/docker-images/mailman.yaml @@ -0,0 +1,29 @@ +- job: + name: system-config-build-image-mailman + description: Build mailman docker images + provides: mailman-container-images + parent: system-config-build-image + vars: &mailman_vars + docker_images: + - context: docker/mailman/core + repository: opendevorg/mailman-core + - context: docker/mailman/web + repository: opendevorg/mailman-web + docker_use_buildkit: true + files: &mailman_files + - docker/mailman/ + +- job: + name: system-config-upload-image-mailman + description: Build and upload mailman images. + provides: mailman-container-images + parent: system-config-upload-image + vars: *mailman_vars + files: *mailman_files + +- job: + name: system-config-promote-image-mailman + description: Promote previously published mailman images to latest. + parent: system-config-promote-image + vars: *mailman_vars + files: *mailman_files diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 4024ed5e94..7b7687d954 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -30,7 +30,11 @@ soft: true - system-config-run-kerberos - system-config-run-lists - - system-config-run-lists3 + - system-config-run-lists3: + dependencies: + - name: opendev-buildset-registry + - name: system-config-build-image-mailman + soft: true - system-config-run-nodepool: dependencies: - name: opendev-buildset-registry @@ -113,6 +117,7 @@ - name: system-config-build-image-python-base-3.9-bullseye soft: true - system-config-build-image-etherpad + - system-config-build-image-mailman - system-config-build-image-gitea: dependencies: - name: opendev-buildset-registry @@ -181,7 +186,11 @@ soft: true - system-config-run-kerberos - system-config-run-lists - - system-config-run-lists3 + - system-config-run-lists3: + dependencies: + - name: opendev-buildset-registry + - name: system-config-upload-image-mailman + soft: true - system-config-run-nodepool: dependencies: - name: opendev-buildset-registry @@ -252,6 +261,7 @@ - system-config-upload-image-hound - system-config-upload-image-assets - system-config-upload-image-etherpad + - system-config-upload-image-mailman - system-config-upload-image-gitea: dependencies: - name: opendev-buildset-registry @@ -303,6 +313,7 @@ - system-config-promote-image-gitea-init - system-config-promote-image-gitea - system-config-promote-image-etherpad + - system-config-promote-image-mailman - system-config-promote-image-haproxy-statsd - system-config-promote-image-zookeeper-statsd - system-config-promote-image-accessbot @@ -469,6 +480,8 @@ soft: true - name: infra-prod-letsencrypt soft: true + - name: system-config-promote-image-mailman + soft: true - infra-prod-service-mirror: &infra-prod-service-mirror dependencies: - name: infra-prod-letsencrypt diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index d428909849..93a0c42e3a 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -290,7 +290,8 @@ name: system-config-run-lists3 # We don't use the system-config-run-containers base job because we # are consuming upstream containers only. - parent: system-config-run + parent: system-config-run-containers + requires: mailman-container-images description: | Run the playbook for a mailman3 list server. timeout: 3600