diff --git a/docker/mailman/core/Dockerfile b/docker/mailman/core/Dockerfile new file mode 100644 index 0000000000..3c825a116c --- /dev/null +++ b/docker/mailman/core/Dockerfile @@ -0,0 +1,37 @@ +# syntax = docker/dockerfile:1.3 +# Use 3.15 for Core since it has Python 3.9 +FROM alpine:3.15 + +#Add startup script to container +COPY docker-entrypoint.sh /usr/local/bin/ + +# Add requirements file. +COPY requirements.txt /tmp/ + +#Install all required packages, add user for executing mailman and set execution rights for startup script +RUN --mount=type=cache,target=/root/.cache \ + apk update \ + && apk add --virtual build-deps gcc python3-dev musl-dev postgresql-dev \ + libffi-dev \ + # Mailman html to plaintext conversion uses lynx. + # psutil needs linux-headers to compile on musl c library. + && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers py-cryptography mariadb-connector-c lynx \ + && python3 -m pip install -U pip setuptools wheel \ + && python3 -m pip install psycopg2 \ + gunicorn==19.9.0 \ + pymysql \ + sqlalchemy \ + -r /tmp/requirements.txt \ + && apk del build-deps \ + && adduser -S mailman + +# Change the working directory. +WORKDIR /opt/mailman + +#Expose the ports for the api (8001) and lmtp (8024) +EXPOSE 8001 8024 + +ENV MAILMAN_CONFIG_FILE /etc/mailman.cfg + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["master", "--force"] diff --git a/docker/mailman/core/README.md b/docker/mailman/core/README.md new file mode 100644 index 0000000000..ce611f4f5f --- /dev/null +++ b/docker/mailman/core/README.md @@ -0,0 +1,6 @@ +This is forked from https://github.com/maxking/docker-mailman/tree/main/core +and you should refer to that repo for information. The upstream repo is MIT +licensed. + +The contents in this dir are based on commit +22b3b7d4024a00e9896837fa04883cbaeef38b20. diff --git a/docker/mailman/core/docker-entrypoint.sh b/docker/mailman/core/docker-entrypoint.sh new file mode 100755 index 0000000000..80aa6c79fa --- /dev/null +++ b/docker/mailman/core/docker-entrypoint.sh @@ -0,0 +1,233 @@ +#! /bin/bash +set -e + +function wait_for_postgres () { + # Check if the postgres database is up and accepting connections before + # moving forward. + # TODO: Use python3's psycopg2 module to do this in python3 instead of + # installing postgres-client in the image. + until psql $DATABASE_URL -c '\l'; do + >&2 echo "Postgres is unavailable - sleeping" + sleep 1 + done + >&2 echo "Postgres is up - continuing" +} + +function wait_for_mysql () { + # Check if MySQL is up and accepting connections. + readarray -d' ' -t ENDPOINT <<< $(python3 -c "from urllib.parse import urlparse; o = urlparse('$DATABASE_URL'); print('%s %s' % (o.hostname, o.port if o.port else '3306'));") + until mysqladmin ping --host ${ENDPOINT[0]} --port ${ENDPOINT[1]} --silent; do + >&2 echo "MySQL is unavailable - sleeping" + sleep 1 + done + >&2 echo "MySQL is up - continuing" +} + +# Empty the config file. +echo "# This file is autogenerated at container startup." > /etc/mailman.cfg + +# Check if $MM_HOSTNAME is set, if not, set it to the value returned by +# `hostname -i` command to set it to whatever IP address is assigned to the +# container. +if [[ ! -v MM_HOSTNAME ]]; then + export MM_HOSTNAME=`hostname -i` +fi + +# SMTP_HOST defaults to the gateway +if [[ ! -v SMTP_HOST ]]; then + export SMTP_HOST=$(/sbin/ip route | awk '/default/ { print $3 }') + echo "SMTP_HOST not specified, using the gateway ($SMTP_HOST) as default" +fi + +if [[ ! -v SMTP_PORT ]]; then + export SMTP_PORT=25 +fi + +# Check if REST port, username, and password are set, if not, set them +# to default values. +if [[ ! -v MAILMAN_REST_PORT ]]; then + export MAILMAN_REST_PORT='8001' +fi + +if [[ ! -v MAILMAN_REST_USER ]]; then + export MAILMAN_REST_USER='restadmin' +fi + +if [[ ! -v MAILMAN_REST_PASSWORD ]]; then + export MAILMAN_REST_PASSWORD='restpass' +fi + +function setup_database () { + if [[ ! -v DATABASE_URL ]] + then + echo "Environment variable DATABASE_URL should be defined..." + exit 1 + fi + + # Translate mysql:// urls to mysql+mysql:// backend: + if [[ "$DATABASE_URL" == mysql://* ]]; then + DATABASE_URL="mysql+pymysql://${DATABASE_URL:8}" + echo "Database URL was automatically rewritten to: $DATABASE_URL" + fi + + # If DATABASE_CLASS is not set, guess it for common databases: + if [ -z "$DATABASE_CLASS" ]; then + if [[ ("$DATABASE_URL" == mysql:*) || + ("$DATABASE_URL" == mysql+*) ]]; then + DATABASE_CLASS=mailman.database.mysql.MySQLDatabase + fi + if [[ ("$DATABASE_URL" == postgres:*) || + ("$DATABASE_URL" == postgres+*) ]]; then + DATABASE_CLASS=mailman.database.postgresql.PostgreSQLDatabase + fi + fi + + cat >> /etc/mailman.cfg <> /etc/mailman.cfg << EOF +[runner.retry] +sleep_time: 10s + +[webservice] +hostname: $MM_HOSTNAME +port: $MAILMAN_REST_PORT +admin_user: $MAILMAN_REST_USER +admin_pass: $MAILMAN_REST_PASSWORD +configuration: /etc/gunicorn.cfg + +EOF + +# Generate a basic gunicorn.cfg. +SITE_DIR=$(python3 -c 'import site; print(site.getsitepackages()[0])') +cp "${SITE_DIR}/mailman/config/gunicorn.cfg" /etc/gunicorn.cfg + +# Generate a basic configuration to use exim +cat > /tmp/exim-mailman.cfg < /etc/postfix-mailman.cfg << EOF +[postfix] +transport_file_type: regex +# While in regex mode, postmap_command is never used, a placeholder +# is added here so that it doesn't break anything. +postmap_command: true +EOF + +# Generate a basic configuration to use postfix. +cat > /tmp/postfix-mailman.cfg <> /etc/mailman.cfg +elif [ "$MTA" == "postfix" ] +then + echo "Using Postfix configuration" + cat /tmp/postfix-mailman.cfg >> /etc/mailman.cfg +else + echo "No MTA environment variable found, defaulting to Exim" + cat /tmp/exim-mailman.cfg >> /etc/mailman.cfg +fi + +rm -f /tmp/{postfix,exim}-mailman.cfg + +if [[ -e /opt/mailman/mailman-extra.cfg ]] +then + echo "Found configuration file at /opt/mailman/mailman-extra.cfg" + cat /opt/mailman/mailman-extra.cfg >> /etc/mailman.cfg +fi + +if [[ -e /opt/mailman/gunicorn-extra.cfg ]] +then + echo "Found [webserver] configuration file at /opt/mailman/gunicorn-extra.cfg" + cat /opt/mailman/gunicorn-extra.cfg > /etc/gunicorn.cfg +fi + +if [[ -v HYPERKITTY_API_KEY ]]; then + +echo "HYPERKITTY_API_KEY found, setting up HyperKitty archiver..." + +cat >> /etc/mailman.cfg << EOF +[archiver.hyperkitty] +class: mailman_hyperkitty.Archiver +enable: yes +configuration: /etc/mailman-hyperkitty.cfg + +EOF + +if [[ ! -v HYPERKITTY_URL ]]; then + echo "HYPERKITTY_URL not set, using the default value of http://mailman-web:8000/hyperkitty" + export HYPERKITTY_URL="http://mailman-web:8000/hyperkitty/" +fi + +# Generate a basic mailman-hyperkitty.cfg. +cat > /etc/mailman-hyperkitty.cfg <&2 echo "Postgres is unavailable - sleeping" + sleep 1 + done + >&2 echo "Postgres is up - continuing" +} + +function wait_for_mysql () { + # Check if MySQL is up and accepting connections. + readarray -d' ' -t ENDPOINT <<< $(python3 -c "from urllib.parse import urlparse; o = urlparse('$DATABASE_URL'); print('%s %s' % (o.hostname, o.port if o.port else '3306'));") + until mysqladmin ping --host ${ENDPOINT[0]} --port ${ENDPOINT[1]} --silent; do + >&2 echo "MySQL is unavailable - sleeping" + sleep 1 + done + >&2 echo "MySQL is up - continuing" +} + +function check_or_create () { + # Check if the path exists, if not, create the directory. + if [[ ! -e dir ]]; then + echo "$1 does not exist, creating ..." + mkdir "$1" + fi +} + +# function postgres_ready(){ +# python << END +# import sys +# import psycopg2 +# try: +# conn = psycopg2.connect(dbname="$POSTGRES_DB", user="$POSTGRES_USER", password="$POSTGRES_PASSWORD", host="postgres") +# except psycopg2.OperationalError: +# sys.exit(-1) +# sys.exit(0) +# END +# } + +# SMTP_HOST defaults to the gateway +if [[ ! -v SMTP_HOST ]]; then + export SMTP_HOST=$(/sbin/ip route | awk '/default/ { print $3 }') +fi + +# Check if $SECRET_KEY is defined, if not, bail out. +if [[ ! -v SECRET_KEY ]]; then + echo "SECRET_KEY is not defined. Aborting." + exit 1 +fi + +# Check if $DATABASE_URL is defined, if not, use a standard sqlite database. +# +# If the $DATABASE_URL is defined and is postgres, check if it is available +# yet. Do not start the container before the postgresql boots up. +# +# If the $DATABASE_URL is defined and is mysql, check if the database is +# available before the container boots up. +# +# TODO: Check the database type and detect if it is up based on that. For now, +# assume that postgres is being used if DATABASE_URL is defined. + +if [[ ! -v DATABASE_URL ]]; then + echo "DATABASE_URL is not defined. Using sqlite database..." + export DATABASE_URL=sqlite://mailmanweb.db + export DATABASE_TYPE='sqlite' +fi + +if [[ "$DATABASE_TYPE" = 'postgres' ]] +then + wait_for_postgres +elif [[ "$DATABASE_TYPE" = 'mysql' ]] +then + wait_for_mysql +fi + +# Check if we are in the correct directory before running commands. +if [[ ! $(pwd) == '/opt/mailman-web' ]]; then + echo "Running in the wrong directory...switching to /opt/mailman-web" + cd /opt/mailman-web +fi + +# Check if the logs directory is setup. +if [[ ! -e /opt/mailman-web-data/logs/mailmanweb.log ]]; then + echo "Creating log file for mailman web" + mkdir -p /opt/mailman-web-data/logs/ + touch /opt/mailman-web-data/logs/mailmanweb.log +fi + +if [[ ! -e /opt/mailman-web-data/logs/uwsgi.log ]]; then + echo "Creating log file for uwsgi.." + touch /opt/mailman-web-data/logs/uwsgi.log +fi + +# Check if the settings_local.py file exists, if yes, copy it too. +if [[ -e /opt/mailman-web-data/settings_local.py ]]; then + echo "Copying settings_local.py ..." + cp /opt/mailman-web-data/settings_local.py /opt/mailman-web/settings_local.py + chown mailman:mailman /opt/mailman-web/settings_local.py +else + echo "settings_local.py not found, it is highly recommended that you provide one" + echo "Using default configuration to run." +fi + +# Collect static for the django installation. +python3 manage.py collectstatic --noinput --clear --verbosity 0 + + +# Compile all the installed po files to mo. +SITE_DIR=$(python3 -c 'import site; print(site.getsitepackages()[0])') +echo "Compiling locale files in $SITE_DIR" +cd $SITE_DIR && /opt/mailman-web/manage.py compilemessages && cd - + +# Compress static files. +python3 manage.py compress --force + + +# Migrate all the data to the database if this is a new installation, otherwise +# this command will upgrade the database. +python3 manage.py migrate + +# If MAILMAN_ADMIN_USER and MAILMAN_ADMIN_EMAIL is defined create a new +# superuser for Django. There is no password setup so it can't login yet unless +# the password is reset. +if [[ -v MAILMAN_ADMIN_USER ]] && [[ -v MAILMAN_ADMIN_EMAIL ]]; +then + echo "Creating admin user $MAILMAN_ADMIN_USER ..." + python3 manage.py createsuperuser --noinput --username "$MAILMAN_ADMIN_USER"\ + --email "$MAILMAN_ADMIN_EMAIL" 2> /dev/null || \ + echo "Superuser $MAILMAN_ADMIN_USER already exists" +fi + +# If SERVE_FROM_DOMAIN is defined then rename the default `example.com` +# domain to the defined domain. +if [[ -v SERVE_FROM_DOMAIN ]]; +then + echo "Setting $SERVE_FROM_DOMAIN as the default domain ..." + python3 manage.py shell -c \ + "from django.contrib.sites.models import Site; Site.objects.filter(domain='example.com').update(domain='$SERVE_FROM_DOMAIN', name='$SERVE_FROM_DOMAIN')" +fi + +# Create a mailman user with the specific UID and GID and do not create home +# directory for it. Also chown the logs directory to write the files. +chown mailman:mailman /opt/mailman-web-data -R + +[[ -v DISKCACHE_PATH ]] && chown mailman:mailman "${DISKCACHE_PATH}" -R + +exec $@ diff --git a/docker/mailman/web/mailman-web/__init__.py b/docker/mailman/web/mailman-web/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/mailman/web/mailman-web/manage.py b/docker/mailman/web/mailman-web/manage.py new file mode 100755 index 0000000000..4eb6f348a4 --- /dev/null +++ b/docker/mailman/web/mailman-web/manage.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") + + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/docker/mailman/web/mailman-web/settings.py b/docker/mailman/web/mailman-web/settings.py new file mode 100644 index 0000000000..6c2d0d0d90 --- /dev/null +++ b/docker/mailman/web/mailman-web/settings.py @@ -0,0 +1,414 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. +# +# This file is part of Mailman Suite. +# +# Mailman Suite is free sofware: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# Mailman Suite is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. + +# You should have received a copy of the GNU General Public License along +# with Mailman Suite. If not, see . +""" +Django Settings for Mailman Suite (hyperkitty + postorius) + +For more information on this file, see +https://docs.djangoproject.com/en/1.8/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/1.8/ref/settings/ +""" + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +import os +import dj_database_url +import sys +from socket import gethostbyname + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.environ.get('SECRET_KEY') + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = False + +ADMINS = ( + ('Mailman Suite Admin', 'root@localhost'), +) + +SITE_ID = 1 + +# Hosts/domain names that are valid for this site; required if DEBUG is False +# See https://docs.djangoproject.com/en/3.1/ref/settings/#allowed-hosts +ALLOWED_HOSTS = [ + "localhost", # Archiving API from Mailman, keep it. + # "lists.your-domain.org", + # Add here all production URLs you may have. + "mailman-web", + gethostbyname("mailman-web"), + os.environ.get('SERVE_FROM_DOMAIN'), + os.environ.get('DJANGO_ALLOWED_HOSTS'), +] + +# Mailman API credentials +MAILMAN_REST_API_URL = os.environ.get('MAILMAN_REST_URL', 'http://mailman-core:8001') +MAILMAN_REST_API_USER = os.environ.get('MAILMAN_REST_USER', 'restadmin') +MAILMAN_REST_API_PASS = os.environ.get('MAILMAN_REST_PASSWORD', 'restpass') +MAILMAN_ARCHIVER_KEY = os.environ.get('HYPERKITTY_API_KEY') +MAILMAN_ARCHIVER_FROM = (os.environ.get('MAILMAN_HOST_IP', gethostbyname(os.environ.get('MAILMAN_HOSTNAME', 'mailman-core'))),) + +# Application definition + +INSTALLED_APPS = [] +DEFAULT_APPS = [ + 'hyperkitty', + 'postorius', + 'django_mailman3', + # Uncomment the next line to enable the admin: + 'django.contrib.admin', + # Uncomment the next line to enable admin documentation: + # 'django.contrib.admindocs', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'rest_framework', + 'django_gravatar', + 'compressor', + 'haystack', + 'django_extensions', + 'django_q', + 'allauth', + 'allauth.account', + 'allauth.socialaccount', +] + +MAILMAN_WEB_SOCIAL_AUTH = [ + 'django_mailman3.lib.auth.fedora', + 'allauth.socialaccount.providers.openid', + 'allauth.socialaccount.providers.github', + 'allauth.socialaccount.providers.gitlab', + 'allauth.socialaccount.providers.google', +] + +MIDDLEWARE = ( + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.middleware.locale.LocaleMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.middleware.security.SecurityMiddleware', + 'django_mailman3.middleware.TimezoneMiddleware', + 'postorius.middleware.PostoriusMiddleware', +) + +ROOT_URLCONF = 'urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.i18n', + 'django.template.context_processors.media', + 'django.template.context_processors.static', + 'django.template.context_processors.tz', + 'django.template.context_processors.csrf', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + 'django_mailman3.context_processors.common', + 'hyperkitty.context_processors.common', + 'postorius.context_processors.postorius', + ], + }, + }, +] + +WSGI_APPLICATION = 'wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/1.8/ref/settings/#databases +# dj_database_url uses $DATABASE_URL environment variable to create a +# django-style-config-dict. +# https://github.com/kennethreitz/dj-database-url +DATABASES = { + 'default': dj_database_url.config(conn_max_age=600) +} + +# If you're behind a proxy, use the X-Forwarded-Host header +# See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host +USE_X_FORWARDED_HOST = True + + +# Password validation +# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + +# Internationalization +# https://docs.djangoproject.com/en/1.8/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + +STATIC_ROOT = '/opt/mailman-web-data/static' + +STATIC_URL = '/static/' + +# Additional locations of static files + + +# List of finder classes that know how to find static files in +# various locations. +STATICFILES_FINDERS = ( + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + 'compressor.finders.CompressorFinder', +) + + +SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer' + +LOGIN_URL = 'account_login' +LOGIN_REDIRECT_URL = 'list_index' +LOGOUT_URL = 'account_logout' + + +# Use SERVE_FROM_DOMAIN as the default domain in the email. +hostname = os.environ.get('SERVE_FROM_DOMAIN', 'localhost.local') +DEFAULT_FROM_EMAIL = 'postorius@{}'.format(hostname) +SERVER_EMAIL = 'root@{}'.format(hostname) + +# Change this when you have a real email backend +EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' +EMAIL_HOST = os.environ.get('SMTP_HOST', '') +EMAIL_PORT = os.environ.get('SMTP_PORT', 25) +EMAIL_HOST_USER = os.environ.get('SMTP_HOST_USER', '') +EMAIL_HOST_PASSWORD = os.environ.get('SMTP_HOST_PASSWORD', '') +EMAIL_USE_TLS = os.environ.get('SMTP_USE_TLS', False) +EMAIL_USE_SSL = os.environ.get('SMTP_USE_SSL', False) + +# Compatibility with Bootstrap 3 +from django.contrib.messages import constants as messages # flake8: noqa +MESSAGE_TAGS = { + messages.ERROR: 'danger' +} + + +# +# Social auth +# +AUTHENTICATION_BACKENDS = ( + 'django.contrib.auth.backends.ModelBackend', + 'allauth.account.auth_backends.AuthenticationBackend', +) + +# Django Allauth +ACCOUNT_AUTHENTICATION_METHOD = "username_email" +ACCOUNT_EMAIL_REQUIRED = True +ACCOUNT_EMAIL_VERIFICATION = "mandatory" +# You probably want https in production, but this is a dev setup file +ACCOUNT_DEFAULT_HTTP_PROTOCOL = "https" +ACCOUNT_UNIQUE_EMAIL = True + +SOCIALACCOUNT_PROVIDERS = { + 'openid': { + 'SERVERS': [ + dict(id='yahoo', + name='Yahoo', + openid_url='http://me.yahoo.com'), + ], + }, + 'google': { + 'SCOPE': ['profile', 'email'], + 'AUTH_PARAMS': {'access_type': 'online'}, + }, + 'facebook': { + 'METHOD': 'oauth2', + 'SCOPE': ['email'], + 'FIELDS': [ + 'email', + 'name', + 'first_name', + 'last_name', + 'locale', + 'timezone', + ], + 'VERSION': 'v2.4', + }, +} + + +# django-compressor +# https://pypi.python.org/pypi/django_compressor +# +COMPRESS_PRECOMPILERS = ( + ('text/less', 'lessc {infile} {outfile}'), + ('text/x-scss', 'sassc -t compressed {infile} {outfile}'), + ('text/x-sass', 'sassc -t compressed {infile} {outfile}'), +) + +# On a production setup, setting COMPRESS_OFFLINE to True will bring a +# significant performance improvement, as CSS files will not need to be +# recompiled on each requests. It means running an additional "compress" +# management command after each code upgrade. +# http://django-compressor.readthedocs.io/en/latest/usage/#offline-compression +# COMPRESS_OFFLINE = True + +# +# Full-text search engine +# +HAYSTACK_CONNECTIONS = { + 'default': { + 'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine', + 'PATH': "/opt/mailman-web-data/fulltext_index", + # You can also use the Xapian engine, it's faster and more accurate, + # but requires another library. + # http://django-haystack.readthedocs.io/en/v2.4.1/installing_search_engines.html#xapian + # Example configuration for Xapian: + #'ENGINE': 'xapian_backend.XapianEngine' + }, +} + +import sys +# A sample logging configuration. The only tangible logging +# performed by this configuration is to send an email to +# the site admins on every HTTP 500 error when DEBUG=False. +# See http://docs.djangoproject.com/en/dev/topics/logging for +# more details on how to customize your logging configuration. +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse' + } + }, + 'handlers': { + 'mail_admins': { + 'level': 'ERROR', + 'filters': ['require_debug_false'], + 'class': 'django.utils.log.AdminEmailHandler' + }, + 'file':{ + 'level': 'INFO', + 'class': 'logging.handlers.RotatingFileHandler', + #'class': 'logging.handlers.WatchedFileHandler', + 'filename': os.environ.get('DJANGO_LOG_URL','/opt/mailman-web-data/logs/mailmanweb.log'), + 'formatter': 'verbose', + }, + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'simple', + 'level': 'INFO', + 'stream': sys.stdout, + }, + }, + 'loggers': { + 'django.request': { + 'handlers': ['mail_admins', 'file'], + 'level': 'INFO', + 'propagate': True, + }, + 'django': { + 'handlers': ['file'], + 'level': 'INFO', + 'propagate': True, + }, + 'hyperkitty': { + 'handlers': ['file'], + 'level': 'INFO', + 'propagate': True, + }, + 'postorius': { + 'handlers': ['file'], + 'level': 'INFO', + 'propagate': True + }, + }, + 'formatters': { + 'verbose': { + 'format': '%(levelname)s %(asctime)s %(process)d %(name)s %(message)s' + }, + 'simple': { + 'format': '%(levelname)s %(message)s' + }, + }, +} + + +if os.environ.get('LOG_TO_CONSOLE') == 'yes': + LOGGING['loggers']['django']['handlers'].append('console') + LOGGING['loggers']['django.request']['handlers'].append('console') + +# HyperKitty-specific +# +# Only display mailing-lists from the same virtual host as the webserver +FILTER_VHOST = False + + +Q_CLUSTER = { + 'timeout': 300, + 'retry': 300, + 'save_limit': 100, + 'orm': 'default', +} + +POSTORIUS_TEMPLATE_BASE_URL = os.environ.get('POSTORIUS_TEMPLATE_BASE_URL', 'http://mailman-web:8000') + +DISKCACHE_PATH = os.environ.get('DISKCACHE_PATH', '/opt/mailman-web-data/diskcache') +DISKCACHE_SIZE = os.environ.get('DISKCACHE_SIZE', 2 ** 30) # 1 gigabyte + +CACHES = { + 'default': { + 'BACKEND': 'diskcache.DjangoCache', + 'LOCATION': DISKCACHE_PATH, + 'OPTIONS': { + 'size_limit': DISKCACHE_SIZE, + }, + }, +} + +try: + from settings_local import * +except ImportError: + pass + +# Compatibility for older installs that override INSTALLED_APPS +if not INSTALLED_APPS: + INSTALLED_APPS = DEFAULT_APPS + MAILMAN_WEB_SOCIAL_AUTH diff --git a/docker/mailman/web/mailman-web/urls.py b/docker/mailman/web/mailman-web/urls.py new file mode 100644 index 0000000000..aa12b25f73 --- /dev/null +++ b/docker/mailman/web/mailman-web/urls.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. +# +# This file is part of Postorius. +# +# Postorius is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# Postorius is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Postorius. If not, see . + +from django.conf.urls import include +from django.contrib import admin +from django.urls import path, reverse_lazy +from django.views.generic import RedirectView + +urlpatterns = [ + path(r'', RedirectView.as_view( + url=reverse_lazy('list_index'), + permanent=True)), + path(r'mailman3/', include('postorius.urls')), + path(r'archives/', include('hyperkitty.urls')), + path(r'', include('django_mailman3.urls')), + path(r'accounts/', include('allauth.urls')), + # Django admin + path(r'admin/', admin.site.urls), +] diff --git a/docker/mailman/web/mailman-web/uwsgi.ini b/docker/mailman/web/mailman-web/uwsgi.ini new file mode 100644 index 0000000000..8e917a34f5 --- /dev/null +++ b/docker/mailman/web/mailman-web/uwsgi.ini @@ -0,0 +1,53 @@ +[uwsgi] +# Port on which uwsgi will be listening. +uwsgi-socket = 0.0.0.0:8080 +http-socket = 0.0.0.0:8000 + +#Enable threading for python +enable-threads = true + +# Setting uwsgi buffer size to what Apache2 supports. +buffer-size = 8190 + +# Move to the directory wher the django files are. +chdir = /opt/mailman-web + +# Use the wsgi file provided with the django project. +wsgi-file = wsgi.py + +# Setup default number of processes and threads per process. +master = true +processes = 2 +threads = 2 + +# Drop privielges and don't run as root. +uid = mailman +gid = mailman + +# Setup the django_q related worker processes. +attach-daemon = ./manage.py qcluster + +# Setup hyperkitty's cron jobs. +# 'minutely' jobs are run hourly for perf reasons. +# See https://github.com/maxking/docker-mailman/issues/327 +unique-cron = 0 -1 -1 -1 -1 ./manage.py runjobs minutely +unique-cron = -15 -1 -1 -1 -1 ./manage.py runjobs quarter_hourly +unique-cron = 0 -1 -1 -1 -1 ./manage.py runjobs hourly +unique-cron = 0 0 -1 -1 -1 ./manage.py runjobs daily +unique-cron = 0 0 1 -1 -1 ./manage.py runjobs monthly +unique-cron = 0 0 -1 -1 0 ./manage.py runjobs weekly +unique-cron = 0 0 1 1 -1 ./manage.py runjobs yearly + +# Setup the request log. +req-logger = file:/opt/mailman-web-data/logs/uwsgi.log + +# Log cron seperately. +logger = cron file:/opt/mailman-web-data/logs/uwsgi-cron.log +log-route = cron uwsgi-cron + +# Log qcluster commands seperately. +logger = qcluster file:/opt/mailman-web-data/logs/uwsgi-qcluster.log +log-route = qcluster uwsgi-daemons + +# Last log and it logs the rest of the stuff. +logger = file:/opt/mailman-web-data/logs/uwsgi-error.log diff --git a/docker/mailman/web/mailman-web/wsgi.py b/docker/mailman/web/mailman-web/wsgi.py new file mode 100755 index 0000000000..8e59cd8545 --- /dev/null +++ b/docker/mailman/web/mailman-web/wsgi.py @@ -0,0 +1,38 @@ +""" +WSGI config for HyperKitty project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/{{ docs_version }}/howto/deployment/wsgi/ +""" + +import os + +# import sys +# import site + +# For some unknown reason, sometimes mod_wsgi fails to set the python paths to +# the virtualenv, with the 'python-path' option. You can do it here too. +# +# # Remember original sys.path. +# prev_sys_path = list(sys.path) +# # Add here, for the settings module +# site.addsitedir(os.path.abspath(os.path.dirname(__file__))) +# # Add the virtualenv +# venv = os.path.join(os.path.abspath(os.path.dirname(__file__)), +# '..', 'lib', 'python2.6', 'site-packages') +# site.addsitedir(venv) +# # Reorder sys.path so new directories at the front. +# new_sys_path = [] +# for item in list(sys.path): +# if item not in prev_sys_path: +# new_sys_path.append(item) +# sys.path.remove(item) +# sys.path[:0] = new_sys_path + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") + +application = get_wsgi_application() diff --git a/docker/mailman/web/requirements.txt b/docker/mailman/web/requirements.txt new file mode 100644 index 0000000000..20dc945d92 --- /dev/null +++ b/docker/mailman/web/requirements.txt @@ -0,0 +1,4 @@ +mailmanclient==3.3.4 +postorius==1.3.7 +hyperkitty==1.3.6 +django-mailman3==1.3.8 \ No newline at end of file diff --git a/playbooks/roles/mailman3/templates/docker-compose.yaml.j2 b/playbooks/roles/mailman3/templates/docker-compose.yaml.j2 index a53ad65412..b0cf4c73b0 100644 --- a/playbooks/roles/mailman3/templates/docker-compose.yaml.j2 +++ b/playbooks/roles/mailman3/templates/docker-compose.yaml.j2 @@ -4,7 +4,7 @@ version: '2' services: mailman-core: - image: docker.io/maxking/mailman-core:0.4 + image: docker.io/opendevorg/mailman-core:latest restart: always container_name: mailman-core volumes: @@ -27,7 +27,7 @@ services: #user: mailman mailman-web: - image: docker.io/maxking/mailman-web:0.4 + image: docker.io/opendevorg/mailman-web:latest restart: always container_name: mailman-web depends_on: diff --git a/playbooks/roles/mailman3/templates/mailman.vhost.j2 b/playbooks/roles/mailman3/templates/mailman.vhost.j2 index 811b61c87c..e689af2f1a 100644 --- a/playbooks/roles/mailman3/templates/mailman.vhost.j2 +++ b/playbooks/roles/mailman3/templates/mailman.vhost.j2 @@ -39,7 +39,7 @@ SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer Alias /static /var/lib/mailman/web-data/static - Alias /favicon.ico /var/lib/mailman/web-data/static/hyperkitty/img/favicon.ico + Alias /favicon.ico /var/lib/mailman/web-data/static/archives/img/favicon.ico Require local @@ -47,8 +47,8 @@ RewriteEngine On RewriteRule "/pipermail/(.*)" "/var/lib/mailman/web-data/mm2archives/%{HTTP_HOST}/public/$1" - RewriteRule "/cgi-bin/mailman/listinfo/(.*)" "https://%{HTTP_HOST}/postorius/lists/$1.%{HTTP_HOST}/" - RewriteRule "/cgi-bin/mailman/listinfo" "https://%{HTTP_HOST}/postorius/lists/" + RewriteRule "/cgi-bin/mailman/listinfo/(.*)" "https://%{HTTP_HOST}/mailman3/lists/$1.%{HTTP_HOST}/" + RewriteRule "/cgi-bin/mailman/listinfo" "https://%{HTTP_HOST}/mailman3/lists/" ProxyPassMatch ^/static/ ! ProxyPass "/" "uwsgi://localhost:8080/" diff --git a/testinfra/test_lists_opendev_org.py b/testinfra/test_lists_opendev_org.py index 5b1ff89fc9..0dcc32834a 100644 --- a/testinfra/test_lists_opendev_org.py +++ b/testinfra/test_lists_opendev_org.py @@ -39,22 +39,22 @@ def test_apache2_listening(host): def test_mailman3_screenshots(host): shots = ( ("https://lists.opendev.org:443", None, "mm3-opendev-main.png"), - ("https://lists.opendev.org:443/hyperkitty/", + ("https://lists.opendev.org:443/archives/", None, "mm3-opendev-archives.png"), ("https://lists.opendev.org:443" - "/hyperkitty/list/service-discuss@lists.opendev.org/", + "/archives/list/service-discuss@lists.opendev.org/", None, "mm3-opendev-list.png"), ("https://lists.opendev.org:443" - "/accounts/login/?next=/postorius/lists/", + "/accounts/login/?next=/mailman3/lists/", None, "mm3-opendev-login.png"), ("https://lists.openstack.org:443", None, "mm3-openstack-main.png"), - ("https://lists.openstack.org:443/hyperkitty/", + ("https://lists.openstack.org:443/archives/", None, "mm3-openstack-archives.png"), ("https://lists.openstack.org:443" - "/hyperkitty/list/openstack-discuss@lists.openstack.org/", + "/archives/list/openstack-discuss@lists.openstack.org/", None, "mm3-openstack-list.png"), ("https://lists.openstack.org:443" - "/accounts/login/?next=/postorius/lists/", + "/accounts/login/?next=/mailman3/lists/", None, "mm3-openstack-login.png"), ) diff --git a/tools/run-bashate.sh b/tools/run-bashate.sh index e57e734c82..b38c4e10e6 100755 --- a/tools/run-bashate.sh +++ b/tools/run-bashate.sh @@ -1,6 +1,8 @@ #!/bin/bash ROOT=$(readlink -fn $(dirname $0)/.. ) -find $ROOT -type f -not -wholename \*.tox/\* -and \( -name \*.sh \ - -or -name \*rc -or -name functions\* \) -print0 \ +find $ROOT -type f -not -wholename \*.tox/\* -and \ + \( -name \*.sh -or -name \*rc -or -name functions\* \) -and \ + -not -path \*/docker/mailman/\* \ + -print0 \ | xargs -0 bashate -i E006,E010 -v diff --git a/zuul.d/docker-images/mailman.yaml b/zuul.d/docker-images/mailman.yaml new file mode 100644 index 0000000000..783d338cb6 --- /dev/null +++ b/zuul.d/docker-images/mailman.yaml @@ -0,0 +1,29 @@ +- job: + name: system-config-build-image-mailman + description: Build mailman docker images + provides: mailman-container-images + parent: system-config-build-image + vars: &mailman_vars + docker_images: + - context: docker/mailman/core + repository: opendevorg/mailman-core + - context: docker/mailman/web + repository: opendevorg/mailman-web + docker_use_buildkit: true + files: &mailman_files + - docker/mailman/ + +- job: + name: system-config-upload-image-mailman + description: Build and upload mailman images. + provides: mailman-container-images + parent: system-config-upload-image + vars: *mailman_vars + files: *mailman_files + +- job: + name: system-config-promote-image-mailman + description: Promote previously published mailman images to latest. + parent: system-config-promote-image + vars: *mailman_vars + files: *mailman_files diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 4024ed5e94..7b7687d954 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -30,7 +30,11 @@ soft: true - system-config-run-kerberos - system-config-run-lists - - system-config-run-lists3 + - system-config-run-lists3: + dependencies: + - name: opendev-buildset-registry + - name: system-config-build-image-mailman + soft: true - system-config-run-nodepool: dependencies: - name: opendev-buildset-registry @@ -113,6 +117,7 @@ - name: system-config-build-image-python-base-3.9-bullseye soft: true - system-config-build-image-etherpad + - system-config-build-image-mailman - system-config-build-image-gitea: dependencies: - name: opendev-buildset-registry @@ -181,7 +186,11 @@ soft: true - system-config-run-kerberos - system-config-run-lists - - system-config-run-lists3 + - system-config-run-lists3: + dependencies: + - name: opendev-buildset-registry + - name: system-config-upload-image-mailman + soft: true - system-config-run-nodepool: dependencies: - name: opendev-buildset-registry @@ -252,6 +261,7 @@ - system-config-upload-image-hound - system-config-upload-image-assets - system-config-upload-image-etherpad + - system-config-upload-image-mailman - system-config-upload-image-gitea: dependencies: - name: opendev-buildset-registry @@ -303,6 +313,7 @@ - system-config-promote-image-gitea-init - system-config-promote-image-gitea - system-config-promote-image-etherpad + - system-config-promote-image-mailman - system-config-promote-image-haproxy-statsd - system-config-promote-image-zookeeper-statsd - system-config-promote-image-accessbot @@ -469,6 +480,8 @@ soft: true - name: infra-prod-letsencrypt soft: true + - name: system-config-promote-image-mailman + soft: true - infra-prod-service-mirror: &infra-prod-service-mirror dependencies: - name: infra-prod-letsencrypt diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index d428909849..93a0c42e3a 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -290,7 +290,8 @@ name: system-config-run-lists3 # We don't use the system-config-run-containers base job because we # are consuming upstream containers only. - parent: system-config-run + parent: system-config-run-containers + requires: mailman-container-images description: | Run the playbook for a mailman3 list server. timeout: 3600