Update Mailman containers to latest versions

This upgrades our images to Alpine 3.20, Django 4.2, Mailman 3.3.10,
Postorius 1.3.13, Hyperkitty 1.3.12 and django-mailman3 1.3.15.
Files are re-synced with upstream (either container or project)
files, with versions and any alterations noted.

Change-Id: I78d37c0635d38ecfc1d1143a69892fe8d8685214

docker/mailman/core/Dockerfile

@@ -1,9 +1,6 @@
 # syntax = docker/dockerfile:1.3
 # Use 3.15 for Core since it has Python 3.9
-FROM alpine:3.18
-
-#Add startup script to container
-COPY docker-entrypoint.sh /usr/local/bin/
+FROM alpine:3.20
 
 # Add requirements file.
 COPY requirements.txt /tmp/
@@ -16,15 +13,17 @@ RUN --mount=type=cache,target=/root/.cache \
   # Mailman html to plaintext conversion uses lynx.
   # psutil needs linux-headers to compile on musl c library.
     && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers py-cryptography mariadb-connector-c lynx tzdata \
-    && python3 -m pip install -U pip setuptools wheel \
-        && python3 -m pip install psycopg2 \
-                   gunicorn==19.9.0 \
+    && python3 -m pip install --break-system-packages -U pip setuptools wheel \
+        && python3 -m pip install --break-system-packages psycopg2 \
                    pymysql \
                    -r /tmp/requirements.txt \
                    'importlib-resources<6.0.0' \
     && apk del build-deps \
     && adduser -S mailman
 
+#Add startup script to container
+COPY docker-entrypoint.sh /usr/local/bin/
+
 # Change the working directory.
 WORKDIR /opt/mailman
 

docker/mailman/core/README.md

@@ -3,4 +3,4 @@ and you should refer to that repo for information. The upstream repo is MIT
 licensed.
 
 The contents in this dir are based on commit
-13411cbe624f5b590f14b801200b7dc107624511.
+2644f1ac37200b13c0112d657d6854bf471bb451.

docker/mailman/core/docker-entrypoint.sh

@@ -43,6 +43,18 @@ if [[ ! -v SMTP_PORT ]]; then
 	export SMTP_PORT=25
 fi
 
+if [[ ! -v SMTP_SECURE_MODE ]]; then
+	export SMTP_SECURE_MODE="smtp"
+fi
+
+if [[ ! -v SMTP_VERIFY_HOSTNAME ]]; then
+	export SMTP_VERIFY_HOSTNAME="true"
+fi
+
+if [[ ! -v SMTP_VERIFY_CERT ]]; then
+	export SMTP_VERIFY_CERT="true"
+fi
+
 # Check if REST port, username, and password are set, if not, set them
 # to default values.
 if [[ ! -v MAILMAN_REST_PORT ]]; then
@@ -144,6 +156,9 @@ smtp_host: $SMTP_HOST
 smtp_port: $SMTP_PORT
 smtp_user: $SMTP_HOST_USER
 smtp_pass: $SMTP_HOST_PASSWORD
+smtp_secure_mode: $SMTP_SECURE_MODE
+smtp_verify_hostname: $SMTP_VERIFY_HOSTNAME
+smtp_verify_cert: $SMTP_VERIFY_CERT
 configuration: python:mailman.config.exim4
 
 EOF
@@ -167,6 +182,9 @@ smtp_host: $SMTP_HOST
 smtp_port: $SMTP_PORT
 smtp_user: $SMTP_HOST_USER
 smtp_pass: $SMTP_HOST_PASSWORD
+smtp_secure_mode: $SMTP_SECURE_MODE
+smtp_verify_hostname: $SMTP_VERIFY_HOSTNAME
+smtp_verify_cert: $SMTP_VERIFY_CERT
 configuration: /etc/postfix-mailman.cfg
 
 EOF
@@ -229,7 +247,17 @@ echo "HYPERKITTY_API_KEY not defined, skipping HyperKitty setup..."
 fi
 
 # Now chown the places where mailman wants to write stuff.
-chown -R mailman /opt/mailman
+VAR_DIR="/opt/mailman/var"
+# Check if the directory exists
+if [ ! -d "$VAR_DIR" ]; then
+  # Directory does not exist, so create it
+  mkdir -p "$VAR_DIR"
+  echo "Directory $VAR_DIR created."
+else
+  echo "Directory $VAR_DIR already exists."
+fi
+
+chown -R mailman $VAR_DIR
 
 # Generate the LMTP files for postfix if needed.
 su-exec mailman mailman aliases

docker/mailman/core/requirements.txt

@@ -1,5 +1,5 @@
 # This is a separate file from Dockerfile so that we can use dependabot
 # for version updates that isn't supported for contents inside the
 # Dockerfile.
-mailman==3.3.9
+mailman==3.3.10
 mailman-hyperkitty==1.2.1

docker/mailman/web/Dockerfile

@@ -1,10 +1,6 @@
 # syntax = docker/dockerfile:1.3
-FROM alpine:3.18.4
+FROM alpine:3.20.3
 
-# Add needed files for uwsgi server + settings for django
-COPY mailman-web /opt/mailman-web
-# Add startup script to container
-COPY docker-entrypoint.sh /usr/local/bin/
 # Add requirements file.
 COPY requirements.txt /tmp/
 
@@ -15,11 +11,11 @@ RUN --mount=type=cache,target=/root/.cache \
     set -ex \
 	&& apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \
            postgresql-dev mariadb-dev mariadb-connector-c python3-dev libffi-dev openldap-dev cargo rust \
-	&& apk add --no-cache --virtual .mailman-rundeps bash sassc tzdata \
+	&& apk add --no-cache --virtual .mailman-rundeps bash sassc tzdata libldap \
 	   postgresql-client mysql-client py3-mysqlclient curl mailcap gettext \
 	   python3 py3-pip xapian-core xapian-bindings-python3 libffi pcre-dev py-cryptography \
-	&& python3 -m pip install -U 'Django<4.2' pip setuptools wheel \
-	&& pip install -r /tmp/requirements.txt \
+	&& python3 -m pip install --break-system-packages -U 'Django<4.3' pip setuptools wheel \
+	&& pip install --break-system-packages -r /tmp/requirements.txt \
 		whoosh \
 		uwsgi \
 		psycopg2 \
@@ -28,14 +24,22 @@ RUN --mount=type=cache,target=/root/.cache \
 		typing \
 		xapian-haystack \
 		django-auth-ldap \
-		python-memcached \
+		pymemcache \
 		diskcache \
 		django-utils-six \
 		tzdata \
+		pytz \
+		'django-allauth[socialaccount,openid]' \
 	&& apk del .build-deps \
 	&& addgroup -S mailman \
-	&& adduser -S -G mailman mailman \
-	&& chown -R mailman /opt/mailman-web/ \
+	&& adduser -S -G mailman mailman
+
+# Add needed files for uwsgi server + settings for django
+COPY mailman-web /opt/mailman-web
+# Add startup script to container
+COPY docker-entrypoint.sh /usr/local/bin/
+
+RUN chown -R mailman /opt/mailman-web/ \
 	&& chmod u+x /opt/mailman-web/manage.py
 
 WORKDIR /opt/mailman-web

docker/mailman/web/README.md

@@ -3,4 +3,4 @@ and you should refer to that repo for information. The upstream repo is MIT
 licensed.
 
 The contents in this dir are based on commit
-13411cbe624f5b590f14b801200b7dc107624511.
+2644f1ac37200b13c0112d657d6854bf471bb451.

docker/mailman/web/docker-entrypoint.sh

@@ -68,7 +68,7 @@ fi
 
 if [[ ! -v DATABASE_URL ]]; then
 	echo "DATABASE_URL is not defined. Using sqlite database..."
-	export DATABASE_URL=sqlite://mailmanweb.db
+	export DATABASE_URL=sqlite:////opt/mail-web-data/mailmanweb.db
 	export DATABASE_TYPE='sqlite'
 fi
 

docker/mailman/web/mailman-web/settings.py

@@ -29,7 +29,7 @@ https://docs.djangoproject.com/en/1.8/ref/settings/
 import os
 import dj_database_url
 import sys
-from socket import gethostbyname
+from socket import gethostbyname, gaierror
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 
@@ -50,9 +50,14 @@ SITE_ID = 1
 ALLOWED_HOSTS = [
     "localhost",  # Archiving API from Mailman, keep it.
     "mailman-web",
-    gethostbyname("mailman-web"),
     os.environ.get('SERVE_FROM_DOMAIN'),
 ]
+
+try:
+    ALLOWED_HOSTS.append(gethostbyname("mailman-web")) # only add if this resolves
+except gaierror:
+    pass
+
 ALLOWED_HOSTS.extend(os.getenv("DJANGO_ALLOWED_HOSTS", "").split(","))
 
 # Mailman API credentials
@@ -79,6 +84,7 @@ DEFAULT_APPS = [
     'django.contrib.sites',
     'django.contrib.messages',
     'django.contrib.staticfiles',
+    'django.contrib.humanize',
     'rest_framework',
     'django_gravatar',
     'compressor',
@@ -150,6 +156,11 @@ DATABASES = {
     'default': dj_database_url.config(conn_max_age=600)
 }
 
+# Avoid Django 3.2+ warning
+# https://github.com/maxking/docker-mailman/issues/595
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+
+
 # If you're behind a proxy, use the X-Forwarded-Host header
 # See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host
 USE_X_FORWARDED_HOST = True

docker/mailman/web/mailman-web/urls.py

@@ -1,7 +1,15 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
 #
-# This file is part of Postorius.
+# This file has been copied from:
+#  https://gitlab.com/mailman/mailman-web/-/blob/5a44178/mailman_web/urls.py
+# In order to match upstream Mailman URL choices (putting modern paths first
+# with fallback routes after for backward compatibility); it can be reverted
+# once https://github.com/maxking/docker-mailman/blob/main/web/mailman-web/ has
+# updated to match.
+#
+# Copyright (C) 2023 by the Free Software Foundation, Inc.
+#
+# This file is part of mailman-web.
 #
 # Postorius is free software: you can redistribute it and/or modify it under
 # the terms of the GNU General Public License as published by the Free
@@ -16,19 +24,25 @@
 # You should have received a copy of the GNU General Public License along with
 # Postorius.  If not, see <http://www.gnu.org/licenses/>.
 
+
 from django.conf.urls import include
 from django.contrib import admin
 from django.urls import path, reverse_lazy
 from django.views.generic import RedirectView
 
 urlpatterns = [
-    path(r'', RedirectView.as_view(
-        url=reverse_lazy('list_index'),
-        permanent=True)),
-    path(r'mailman3/', include('postorius.urls')),
-    path(r'archives/', include('hyperkitty.urls')),
-    path(r'', include('django_mailman3.urls')),
-    path(r'accounts/', include('allauth.urls')),
-    # Django admin
-    path(r'admin/', admin.site.urls),
+    path(
+        '',
+        RedirectView.as_view(url=reverse_lazy('list_index'), permanent=True),
+    ),
+    # Include alternate Postorius and HyperKitty URLs.
+    path('postorius/', include('postorius.urls')),
+    path('hyperkitty/', include('hyperkitty.urls')),
+    # Order counts for various links. Put the above first and the following
+    # after so the suggested Apache config still works.
+    path('mailman3/', include('postorius.urls')),
+    path('archives/', include('hyperkitty.urls')),
+    path('', include('django_mailman3.urls')),
+    path('accounts/', include('allauth.urls')),
+    path('admin/', admin.site.urls),
 ]

docker/mailman/web/mailman-web/uwsgi.ini

@@ -3,13 +3,13 @@
 uwsgi-socket = 0.0.0.0:8080
 http-socket = 0.0.0.0:8000
 
-#Enable threading for python
+# Enable threading for python
 enable-threads = true
 
 # Setting uwsgi buffer size to what Apache2 supports.
 buffer-size = 8190
 
-# Move to the directory wher the django files are.
+# Move to the directory where the django files are.
 chdir = /opt/mailman-web
 
 # Use the wsgi file provided with the django project.
@@ -20,7 +20,7 @@ master = true
 processes = 2
 threads = 2
 
-# Drop privielges and don't run as root.
+# Drop privileges and don't run as root.
 uid = mailman
 gid = mailman
 

docker/mailman/web/requirements.txt

@@ -1,4 +1,4 @@
 mailmanclient==3.3.5
-postorius==1.3.10
-hyperkitty==1.3.8
-django-mailman3==1.3.11
+postorius==1.3.13
+hyperkitty==1.3.12
+django-mailman3==1.3.15

playbooks/roles/mailman3/files/web-settings.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 #
 # This file has been copied from:
-#  https://github.com/maxking/docker-mailman/blob/ab8dfb3/web/mailman-web/settings.py
-# In order to override the ALLOWED_HOSTS, SITE_ID and ADMINS settings.
+#  https://github.com/maxking/docker-mailman/blob/e2dee28/web/mailman-web/settings.py
+# In order to override the ADMINS, SITE_ID and ALLOWED_HOSTS settings.
 #
 # Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
 #
@@ -34,7 +34,7 @@ https://docs.djangoproject.com/en/1.8/ref/settings/
 import os
 import dj_database_url
 import sys
-from socket import gethostbyname
+from socket import gethostbyname, gaierror
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 
@@ -57,10 +57,15 @@ SITE_ID = 0
 ALLOWED_HOSTS = [
     "localhost",  # Archiving API from Mailman, keep it.
     "127.0.0.1",  # OpenDev addition because we use host networking
-    #"mailman-web", # OpenDev edit, won't resolve for us
-    #gethostbyname("mailman-web"), # OpenDev edit, won't resolve for us
+    "mailman-web",
     os.environ.get('SERVE_FROM_DOMAIN'),
 ]
+
+try:
+    ALLOWED_HOSTS.append(gethostbyname("mailman-web")) # only add if this resolves
+except gaierror:
+    pass
+
 ALLOWED_HOSTS.extend(os.getenv("DJANGO_ALLOWED_HOSTS", "").split(","))
 
 # Mailman API credentials
@@ -87,6 +92,7 @@ DEFAULT_APPS = [
     'django.contrib.sites',
     'django.contrib.messages',
     'django.contrib.staticfiles',
+    'django.contrib.humanize',
     'rest_framework',
     'django_gravatar',
     'compressor',
@@ -158,6 +164,11 @@ DATABASES = {
     'default': dj_database_url.config(conn_max_age=600)
 }
 
+# Avoid Django 3.2+ warning
+# https://github.com/maxking/docker-mailman/issues/595
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+
+
 # If you're behind a proxy, use the X-Forwarded-Host header
 # See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host
 USE_X_FORWARDED_HOST = True

playbooks/roles/mailman3/files/web-settings_local.py

@@ -17,9 +17,3 @@ HYPERKITTY_ENABLE_GRAVATAR = False
 MAILMAN_WEB_SOCIAL_AUTH = []
 
 FILTER_VHOST = True
-
-# See
-#  https://docs.djangoproject.com/en/3.2/releases/3.2/#customizing-type-of-auto-created-primary-keys
-# We set this to AutoField to avoid unwanted migrations as we've
-# already created the models with the smaller field size.
-DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'