Upgrade to latest Mailman 3 releases

New releases info:

https://lists.mailman3.org/archives/list/mailman-users@mailman3.org/message/YDUYHVWBGBLASJHAYE2PQKY2F7PNEIKP/

Sync all our forked files from mailman-docker to the current
upstream state, except for our overridden hyperkitty->archives and
postorius->mailman3 URL routes in
docker/mailman/web/mailman-web/urls.py.

Change-Id: I26f45b0f019b9212c148f145a5c36ed777fb111c

docker/mailman/core/Dockerfile

@@ -1,6 +1,6 @@
 # syntax = docker/dockerfile:1.3
 # Use 3.15 for Core since it has Python 3.9
-FROM alpine:3.15
+FROM alpine:3.18
 
 #Add startup script to container
 COPY docker-entrypoint.sh /usr/local/bin/
@@ -15,12 +15,11 @@ RUN --mount=type=cache,target=/root/.cache \
        libffi-dev \
   # Mailman html to plaintext conversion uses lynx.
   # psutil needs linux-headers to compile on musl c library.
-    && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers py-cryptography mariadb-connector-c lynx \
+    && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers py-cryptography mariadb-connector-c lynx tzdata \
     && python3 -m pip install -U pip setuptools wheel \
         && python3 -m pip install psycopg2 \
                    gunicorn==19.9.0 \
                    pymysql \
-                   sqlalchemy \
                    -r /tmp/requirements.txt \
     && apk del build-deps \
     && adduser -S mailman

docker/mailman/core/README.md

@@ -3,4 +3,4 @@ and you should refer to that repo for information. The upstream repo is MIT
 licensed.
 
 The contents in this dir are based on commit
-22b3b7d4024a00e9896837fa04883cbaeef38b20.
+13411cbe624f5b590f14b801200b7dc107624511.

docker/mailman/core/docker-entrypoint.sh

@@ -6,7 +6,7 @@ function wait_for_postgres () {
 	# moving forward.
 	# TODO: Use python3's psycopg2 module to do this in python3 instead of
 	# installing postgres-client in the image.
-	until psql $DATABASE_URL -c '\l'; do
+	until psql -P pager=off $DATABASE_URL -c '\l'; do
 		>&2 echo "Postgres is unavailable - sleeping"
 		sleep 1
 	done
@@ -142,6 +142,8 @@ lmtp_host: $MM_HOSTNAME
 lmtp_port: 8024
 smtp_host: $SMTP_HOST
 smtp_port: $SMTP_PORT
+smtp_user: $SMTP_HOST_USER
+smtp_pass: $SMTP_HOST_PASSWORD
 configuration: python:mailman.config.exim4
 
 EOF
@@ -163,6 +165,8 @@ lmtp_host: $MM_HOSTNAME
 lmtp_port: 8024
 smtp_host: $SMTP_HOST
 smtp_port: $SMTP_PORT
+smtp_user: $SMTP_HOST_USER
+smtp_pass: $SMTP_HOST_PASSWORD
 configuration: /etc/postfix-mailman.cfg
 
 EOF

docker/mailman/core/requirements.txt

@@ -1,8 +1,8 @@
 # This is a separate file from Dockerfile so that we can use dependabot
 # for version updates that isn't supported for contents inside the
 # Dockerfile.
-mailman==3.3.7
-mailman-hyperkitty==1.2.0
+mailman==3.3.8
+mailman-hyperkitty==1.2.1
 # mailman/config/config.py "from importlib_resources import path" ImportError
 # remove when https://gitlab.com/mailman/mailman/-/issues/1093 is solved
 importlib_resources<6

docker/mailman/web/Dockerfile

@@ -1,5 +1,5 @@
 # syntax = docker/dockerfile:1.3
-FROM alpine:3.16.2
+FROM alpine:3.18.2
 
 # Add needed files for uwsgi server + settings for django
 COPY mailman-web /opt/mailman-web
@@ -15,14 +15,14 @@ RUN --mount=type=cache,target=/root/.cache \
     set -ex \
 	&& apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \
            postgresql-dev mariadb-dev mariadb-connector-c python3-dev libffi-dev openldap-dev cargo rust \
-	&& apk add --no-cache --virtual .mailman-rundeps bash sassc \
+	&& apk add --no-cache --virtual .mailman-rundeps bash sassc tzdata \
 	   postgresql-client mysql-client py3-mysqlclient curl mailcap gettext \
 	   python3 py3-pip xapian-core xapian-bindings-python3 libffi pcre-dev py-cryptography \
-	&& python3 -m pip install -U 'Django<4.1' pip setuptools wheel \
+	&& python3 -m pip install -U 'Django<4.2' pip setuptools wheel \
 	&& pip install -r /tmp/requirements.txt \
 		whoosh \
 		uwsgi \
-		'psycopg2<2.9' \
+		psycopg2 \
 		dj-database-url \
 		mysqlclient \
 		typing \

docker/mailman/web/README.md

@@ -3,4 +3,4 @@ and you should refer to that repo for information. The upstream repo is MIT
 licensed.
 
 The contents in this dir are based on commit
-22b3b7d4024a00e9896837fa04883cbaeef38b20.
+13411cbe624f5b590f14b801200b7dc107624511.

docker/mailman/web/docker-entrypoint.sh

@@ -7,7 +7,7 @@ function wait_for_postgres () {
 	# moving forward.
 	# TODO: Use python's psycopg2 module to do this in python instead of
 	# installing postgres-client in the image.
-	until psql $DATABASE_URL -c '\l'; do
+	until psql -P pager=off $DATABASE_URL -c '\l'; do
 		>&2 echo "Postgres is unavailable - sleeping"
 		sleep 1
 	done

docker/mailman/web/mailman-web/settings.py

@@ -49,13 +49,11 @@ SITE_ID = 1
 # See https://docs.djangoproject.com/en/3.1/ref/settings/#allowed-hosts
 ALLOWED_HOSTS = [
     "localhost",  # Archiving API from Mailman, keep it.
-    # "lists.your-domain.org",
-    # Add here all production URLs you may have.
     "mailman-web",
     gethostbyname("mailman-web"),
     os.environ.get('SERVE_FROM_DOMAIN'),
-    os.environ.get('DJANGO_ALLOWED_HOSTS'),
 ]
+ALLOWED_HOSTS.extend(os.getenv("DJANGO_ALLOWED_HOSTS", "").split(","))
 
 # Mailman API credentials
 MAILMAN_REST_API_URL = os.environ.get('MAILMAN_REST_URL', 'http://mailman-core:8001')
@@ -178,7 +176,7 @@ AUTH_PASSWORD_VALIDATORS = [
 
 LANGUAGE_CODE = 'en-us'
 
-TIME_ZONE = 'UTC'
+TIME_ZONE = os.environ.get('TZ', 'UTC')
 
 USE_I18N = True
 

docker/mailman/web/requirements.txt

@@ -1,4 +1,4 @@
-mailmanclient==3.3.4
-postorius==1.3.7
-hyperkitty==1.3.6
-django-mailman3==1.3.8
+mailmanclient==3.3.5
+postorius==1.3.8
+hyperkitty==1.3.7
+django-mailman3==1.3.9