From 24acb98c8d5ecaa2758b4cf1b0f796df849f2124 Mon Sep 17 00:00:00 2001 From: Jeremy Stanley Date: Mon, 23 Sep 2024 19:05:40 +0000 Subject: [PATCH] Update Mailman containers to latest versions This upgrades our images to Alpine 3.20, Django 4.2, Mailman 3.3.10, Postorius 1.3.13, Hyperkitty 1.3.12 and django-mailman3 1.3.15. Files are re-synced with upstream (either container or project) files, with versions and any alterations noted. Change-Id: I78d37c0635d38ecfc1d1143a69892fe8d8685214 --- docker/mailman/core/Dockerfile | 13 ++++--- docker/mailman/core/README.md | 2 +- docker/mailman/core/docker-entrypoint.sh | 30 +++++++++++++++- docker/mailman/core/requirements.txt | 4 +-- docker/mailman/web/Dockerfile | 26 ++++++++------ docker/mailman/web/README.md | 2 +- docker/mailman/web/docker-entrypoint.sh | 2 +- docker/mailman/web/mailman-web/settings.py | 15 ++++++-- docker/mailman/web/mailman-web/urls.py | 36 +++++++++++++------ docker/mailman/web/mailman-web/uwsgi.ini | 6 ++-- docker/mailman/web/requirements.txt | 6 ++-- .../roles/mailman3/files/web-settings.py | 21 ++++++++--- .../mailman3/files/web-settings_local.py | 6 ---- 13 files changed, 115 insertions(+), 54 deletions(-) diff --git a/docker/mailman/core/Dockerfile b/docker/mailman/core/Dockerfile index aa65927941..886305f884 100644 --- a/docker/mailman/core/Dockerfile +++ b/docker/mailman/core/Dockerfile @@ -1,9 +1,6 @@ # syntax = docker/dockerfile:1.3 # Use 3.15 for Core since it has Python 3.9 -FROM alpine:3.18 - -#Add startup script to container -COPY docker-entrypoint.sh /usr/local/bin/ +FROM alpine:3.20 # Add requirements file. COPY requirements.txt /tmp/ @@ -16,15 +13,17 @@ RUN --mount=type=cache,target=/root/.cache \ # Mailman html to plaintext conversion uses lynx. # psutil needs linux-headers to compile on musl c library. && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers py-cryptography mariadb-connector-c lynx tzdata \ - && python3 -m pip install -U pip setuptools wheel \ - && python3 -m pip install psycopg2 \ - gunicorn==19.9.0 \ + && python3 -m pip install --break-system-packages -U pip setuptools wheel \ + && python3 -m pip install --break-system-packages psycopg2 \ pymysql \ -r /tmp/requirements.txt \ 'importlib-resources<6.0.0' \ && apk del build-deps \ && adduser -S mailman +#Add startup script to container +COPY docker-entrypoint.sh /usr/local/bin/ + # Change the working directory. WORKDIR /opt/mailman diff --git a/docker/mailman/core/README.md b/docker/mailman/core/README.md index 8b23dcd8ce..8db70bc65b 100644 --- a/docker/mailman/core/README.md +++ b/docker/mailman/core/README.md @@ -3,4 +3,4 @@ and you should refer to that repo for information. The upstream repo is MIT licensed. The contents in this dir are based on commit -13411cbe624f5b590f14b801200b7dc107624511. +2644f1ac37200b13c0112d657d6854bf471bb451. diff --git a/docker/mailman/core/docker-entrypoint.sh b/docker/mailman/core/docker-entrypoint.sh index d0bd089b0f..519ad2f5b6 100755 --- a/docker/mailman/core/docker-entrypoint.sh +++ b/docker/mailman/core/docker-entrypoint.sh @@ -43,6 +43,18 @@ if [[ ! -v SMTP_PORT ]]; then export SMTP_PORT=25 fi +if [[ ! -v SMTP_SECURE_MODE ]]; then + export SMTP_SECURE_MODE="smtp" +fi + +if [[ ! -v SMTP_VERIFY_HOSTNAME ]]; then + export SMTP_VERIFY_HOSTNAME="true" +fi + +if [[ ! -v SMTP_VERIFY_CERT ]]; then + export SMTP_VERIFY_CERT="true" +fi + # Check if REST port, username, and password are set, if not, set them # to default values. if [[ ! -v MAILMAN_REST_PORT ]]; then @@ -144,6 +156,9 @@ smtp_host: $SMTP_HOST smtp_port: $SMTP_PORT smtp_user: $SMTP_HOST_USER smtp_pass: $SMTP_HOST_PASSWORD +smtp_secure_mode: $SMTP_SECURE_MODE +smtp_verify_hostname: $SMTP_VERIFY_HOSTNAME +smtp_verify_cert: $SMTP_VERIFY_CERT configuration: python:mailman.config.exim4 EOF @@ -167,6 +182,9 @@ smtp_host: $SMTP_HOST smtp_port: $SMTP_PORT smtp_user: $SMTP_HOST_USER smtp_pass: $SMTP_HOST_PASSWORD +smtp_secure_mode: $SMTP_SECURE_MODE +smtp_verify_hostname: $SMTP_VERIFY_HOSTNAME +smtp_verify_cert: $SMTP_VERIFY_CERT configuration: /etc/postfix-mailman.cfg EOF @@ -229,7 +247,17 @@ echo "HYPERKITTY_API_KEY not defined, skipping HyperKitty setup..." fi # Now chown the places where mailman wants to write stuff. -chown -R mailman /opt/mailman +VAR_DIR="/opt/mailman/var" +# Check if the directory exists +if [ ! -d "$VAR_DIR" ]; then + # Directory does not exist, so create it + mkdir -p "$VAR_DIR" + echo "Directory $VAR_DIR created." +else + echo "Directory $VAR_DIR already exists." +fi + +chown -R mailman $VAR_DIR # Generate the LMTP files for postfix if needed. su-exec mailman mailman aliases diff --git a/docker/mailman/core/requirements.txt b/docker/mailman/core/requirements.txt index b100b911b6..b7ff399e5b 100644 --- a/docker/mailman/core/requirements.txt +++ b/docker/mailman/core/requirements.txt @@ -1,5 +1,5 @@ # This is a separate file from Dockerfile so that we can use dependabot # for version updates that isn't supported for contents inside the # Dockerfile. -mailman==3.3.9 -mailman-hyperkitty==1.2.1 \ No newline at end of file +mailman==3.3.10 +mailman-hyperkitty==1.2.1 diff --git a/docker/mailman/web/Dockerfile b/docker/mailman/web/Dockerfile index c440f8ece5..4d3d66f935 100644 --- a/docker/mailman/web/Dockerfile +++ b/docker/mailman/web/Dockerfile @@ -1,10 +1,6 @@ # syntax = docker/dockerfile:1.3 -FROM alpine:3.18.4 +FROM alpine:3.20.3 -# Add needed files for uwsgi server + settings for django -COPY mailman-web /opt/mailman-web -# Add startup script to container -COPY docker-entrypoint.sh /usr/local/bin/ # Add requirements file. COPY requirements.txt /tmp/ @@ -15,11 +11,11 @@ RUN --mount=type=cache,target=/root/.cache \ set -ex \ && apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \ postgresql-dev mariadb-dev mariadb-connector-c python3-dev libffi-dev openldap-dev cargo rust \ - && apk add --no-cache --virtual .mailman-rundeps bash sassc tzdata \ + && apk add --no-cache --virtual .mailman-rundeps bash sassc tzdata libldap \ postgresql-client mysql-client py3-mysqlclient curl mailcap gettext \ python3 py3-pip xapian-core xapian-bindings-python3 libffi pcre-dev py-cryptography \ - && python3 -m pip install -U 'Django<4.2' pip setuptools wheel \ - && pip install -r /tmp/requirements.txt \ + && python3 -m pip install --break-system-packages -U 'Django<4.3' pip setuptools wheel \ + && pip install --break-system-packages -r /tmp/requirements.txt \ whoosh \ uwsgi \ psycopg2 \ @@ -28,14 +24,22 @@ RUN --mount=type=cache,target=/root/.cache \ typing \ xapian-haystack \ django-auth-ldap \ - python-memcached \ + pymemcache \ diskcache \ django-utils-six \ tzdata \ + pytz \ + 'django-allauth[socialaccount,openid]' \ && apk del .build-deps \ && addgroup -S mailman \ - && adduser -S -G mailman mailman \ - && chown -R mailman /opt/mailman-web/ \ + && adduser -S -G mailman mailman + +# Add needed files for uwsgi server + settings for django +COPY mailman-web /opt/mailman-web +# Add startup script to container +COPY docker-entrypoint.sh /usr/local/bin/ + +RUN chown -R mailman /opt/mailman-web/ \ && chmod u+x /opt/mailman-web/manage.py WORKDIR /opt/mailman-web diff --git a/docker/mailman/web/README.md b/docker/mailman/web/README.md index 1781394632..cb4daa098e 100644 --- a/docker/mailman/web/README.md +++ b/docker/mailman/web/README.md @@ -3,4 +3,4 @@ and you should refer to that repo for information. The upstream repo is MIT licensed. The contents in this dir are based on commit -13411cbe624f5b590f14b801200b7dc107624511. +2644f1ac37200b13c0112d657d6854bf471bb451. diff --git a/docker/mailman/web/docker-entrypoint.sh b/docker/mailman/web/docker-entrypoint.sh index 3324d0556e..c00f015786 100755 --- a/docker/mailman/web/docker-entrypoint.sh +++ b/docker/mailman/web/docker-entrypoint.sh @@ -68,7 +68,7 @@ fi if [[ ! -v DATABASE_URL ]]; then echo "DATABASE_URL is not defined. Using sqlite database..." - export DATABASE_URL=sqlite://mailmanweb.db + export DATABASE_URL=sqlite:////opt/mail-web-data/mailmanweb.db export DATABASE_TYPE='sqlite' fi diff --git a/docker/mailman/web/mailman-web/settings.py b/docker/mailman/web/mailman-web/settings.py index 4903645652..c19a827e65 100644 --- a/docker/mailman/web/mailman-web/settings.py +++ b/docker/mailman/web/mailman-web/settings.py @@ -29,7 +29,7 @@ https://docs.djangoproject.com/en/1.8/ref/settings/ import os import dj_database_url import sys -from socket import gethostbyname +from socket import gethostbyname, gaierror BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -50,9 +50,14 @@ SITE_ID = 1 ALLOWED_HOSTS = [ "localhost", # Archiving API from Mailman, keep it. "mailman-web", - gethostbyname("mailman-web"), os.environ.get('SERVE_FROM_DOMAIN'), ] + +try: + ALLOWED_HOSTS.append(gethostbyname("mailman-web")) # only add if this resolves +except gaierror: + pass + ALLOWED_HOSTS.extend(os.getenv("DJANGO_ALLOWED_HOSTS", "").split(",")) # Mailman API credentials @@ -79,6 +84,7 @@ DEFAULT_APPS = [ 'django.contrib.sites', 'django.contrib.messages', 'django.contrib.staticfiles', + 'django.contrib.humanize', 'rest_framework', 'django_gravatar', 'compressor', @@ -150,6 +156,11 @@ DATABASES = { 'default': dj_database_url.config(conn_max_age=600) } +# Avoid Django 3.2+ warning +# https://github.com/maxking/docker-mailman/issues/595 +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' + + # If you're behind a proxy, use the X-Forwarded-Host header # See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host USE_X_FORWARDED_HOST = True diff --git a/docker/mailman/web/mailman-web/urls.py b/docker/mailman/web/mailman-web/urls.py index aa12b25f73..71ac6f0fc8 100644 --- a/docker/mailman/web/mailman-web/urls.py +++ b/docker/mailman/web/mailman-web/urls.py @@ -1,7 +1,15 @@ # -*- coding: utf-8 -*- -# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. # -# This file is part of Postorius. +# This file has been copied from: +# https://gitlab.com/mailman/mailman-web/-/blob/5a44178/mailman_web/urls.py +# In order to match upstream Mailman URL choices (putting modern paths first +# with fallback routes after for backward compatibility); it can be reverted +# once https://github.com/maxking/docker-mailman/blob/main/web/mailman-web/ has +# updated to match. +# +# Copyright (C) 2023 by the Free Software Foundation, Inc. +# +# This file is part of mailman-web. # # Postorius is free software: you can redistribute it and/or modify it under # the terms of the GNU General Public License as published by the Free @@ -16,19 +24,25 @@ # You should have received a copy of the GNU General Public License along with # Postorius. If not, see . + from django.conf.urls import include from django.contrib import admin from django.urls import path, reverse_lazy from django.views.generic import RedirectView urlpatterns = [ - path(r'', RedirectView.as_view( - url=reverse_lazy('list_index'), - permanent=True)), - path(r'mailman3/', include('postorius.urls')), - path(r'archives/', include('hyperkitty.urls')), - path(r'', include('django_mailman3.urls')), - path(r'accounts/', include('allauth.urls')), - # Django admin - path(r'admin/', admin.site.urls), + path( + '', + RedirectView.as_view(url=reverse_lazy('list_index'), permanent=True), + ), + # Include alternate Postorius and HyperKitty URLs. + path('postorius/', include('postorius.urls')), + path('hyperkitty/', include('hyperkitty.urls')), + # Order counts for various links. Put the above first and the following + # after so the suggested Apache config still works. + path('mailman3/', include('postorius.urls')), + path('archives/', include('hyperkitty.urls')), + path('', include('django_mailman3.urls')), + path('accounts/', include('allauth.urls')), + path('admin/', admin.site.urls), ] diff --git a/docker/mailman/web/mailman-web/uwsgi.ini b/docker/mailman/web/mailman-web/uwsgi.ini index 8e917a34f5..897ebade79 100644 --- a/docker/mailman/web/mailman-web/uwsgi.ini +++ b/docker/mailman/web/mailman-web/uwsgi.ini @@ -3,13 +3,13 @@ uwsgi-socket = 0.0.0.0:8080 http-socket = 0.0.0.0:8000 -#Enable threading for python +# Enable threading for python enable-threads = true # Setting uwsgi buffer size to what Apache2 supports. buffer-size = 8190 -# Move to the directory wher the django files are. +# Move to the directory where the django files are. chdir = /opt/mailman-web # Use the wsgi file provided with the django project. @@ -20,7 +20,7 @@ master = true processes = 2 threads = 2 -# Drop privielges and don't run as root. +# Drop privileges and don't run as root. uid = mailman gid = mailman diff --git a/docker/mailman/web/requirements.txt b/docker/mailman/web/requirements.txt index c157bc651c..1765401d6e 100644 --- a/docker/mailman/web/requirements.txt +++ b/docker/mailman/web/requirements.txt @@ -1,4 +1,4 @@ mailmanclient==3.3.5 -postorius==1.3.10 -hyperkitty==1.3.8 -django-mailman3==1.3.11 \ No newline at end of file +postorius==1.3.13 +hyperkitty==1.3.12 +django-mailman3==1.3.15 \ No newline at end of file diff --git a/playbooks/roles/mailman3/files/web-settings.py b/playbooks/roles/mailman3/files/web-settings.py index 415c16d24b..292b333b27 100644 --- a/playbooks/roles/mailman3/files/web-settings.py +++ b/playbooks/roles/mailman3/files/web-settings.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # # This file has been copied from: -# https://github.com/maxking/docker-mailman/blob/ab8dfb3/web/mailman-web/settings.py -# In order to override the ALLOWED_HOSTS, SITE_ID and ADMINS settings. +# https://github.com/maxking/docker-mailman/blob/e2dee28/web/mailman-web/settings.py +# In order to override the ADMINS, SITE_ID and ALLOWED_HOSTS settings. # # Copyright (C) 1998-2016 by the Free Software Foundation, Inc. # @@ -34,7 +34,7 @@ https://docs.djangoproject.com/en/1.8/ref/settings/ import os import dj_database_url import sys -from socket import gethostbyname +from socket import gethostbyname, gaierror BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -57,10 +57,15 @@ SITE_ID = 0 ALLOWED_HOSTS = [ "localhost", # Archiving API from Mailman, keep it. "127.0.0.1", # OpenDev addition because we use host networking - #"mailman-web", # OpenDev edit, won't resolve for us - #gethostbyname("mailman-web"), # OpenDev edit, won't resolve for us + "mailman-web", os.environ.get('SERVE_FROM_DOMAIN'), ] + +try: + ALLOWED_HOSTS.append(gethostbyname("mailman-web")) # only add if this resolves +except gaierror: + pass + ALLOWED_HOSTS.extend(os.getenv("DJANGO_ALLOWED_HOSTS", "").split(",")) # Mailman API credentials @@ -87,6 +92,7 @@ DEFAULT_APPS = [ 'django.contrib.sites', 'django.contrib.messages', 'django.contrib.staticfiles', + 'django.contrib.humanize', 'rest_framework', 'django_gravatar', 'compressor', @@ -158,6 +164,11 @@ DATABASES = { 'default': dj_database_url.config(conn_max_age=600) } +# Avoid Django 3.2+ warning +# https://github.com/maxking/docker-mailman/issues/595 +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' + + # If you're behind a proxy, use the X-Forwarded-Host header # See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host USE_X_FORWARDED_HOST = True diff --git a/playbooks/roles/mailman3/files/web-settings_local.py b/playbooks/roles/mailman3/files/web-settings_local.py index 05ad1eeea7..6475649782 100644 --- a/playbooks/roles/mailman3/files/web-settings_local.py +++ b/playbooks/roles/mailman3/files/web-settings_local.py @@ -17,9 +17,3 @@ HYPERKITTY_ENABLE_GRAVATAR = False MAILMAN_WEB_SOCIAL_AUTH = [] FILTER_VHOST = True - -# See -# https://docs.djangoproject.com/en/3.2/releases/3.2/#customizing-type-of-auto-created-primary-keys -# We set this to AutoField to avoid unwanted migrations as we've -# already created the models with the smaller field size. -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'