opendev/system-config
Code Issues Proposed changes

Merge "Add a mailman3 list server"

Browse Source
This commit is contained in:
Zuul 2022-11-22 18:00:30 +00:00 committed by Gerrit Code Review
commit b7b2157133
24 changed files with 1836 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
inventory/service/group_vars/mailman3.yaml Normal file
View File

@ -0,0 +1,89 @@
exim_queue_interval: '1m'
exim_queue_run_max: '50'
exim_smtp_accept_max: '100'
exim_smtp_accept_max_per_host: '10'
iptables_extra_public_tcp_ports:
- 25
- 80
- 443
- 465
letsencrypt_certs:
lists-opendev-org-main:
- "{{ inventory_hostname }}"
- lists.opendev.org
- lists.airshipit.org
- lists.katacontainers.io
- lists.openinfra.dev
- lists.openstack.org
- lists.starlingx.io
- lists.zuul-ci.org
borg_backup_excludes_extra:
# db is backed up in dumps, don't capture live files
- /var/lib/mailman/database
# backed up by streaming backup
- /var/backups/mailman-mariadb
# Can regenerate indexes from source email files
- /var/lib/mailman/web-data/fulltext_index
exim_routers:
- mailman_verp_router: |
{% raw -%}
driver = dnslookup
condition = ${if or{{eq{$sender_host_address}{127.0.0.1}}\
{eq{$sender_host_address}{::1}}}{yes}{no}}
{% endraw %}
domains = !+local_domains
ignore_target_hosts = <; 0.0.0.0; \
127.0.0.0/8; \
::1/128;fe80::/10;fe \
c0::/10;ff00::/8
senders = "*-bounces@*"
transport = mailman_verp_smtp
- dnslookup: '{{ exim_dnslookup_router }}'
- system_aliases: '{{ exim_system_aliases_router }}'
- domain_aliases: |
driver = redirect
allow_fail
allow_defer
data = ${lookup{$local_part@$domain}lsearch{/etc/aliases.domain}}
file_transport = address_file
pipe_transport = address_pipe
- localuser: '{{ exim_localuser_router }}'
- mailman_copy: |
driver = accept
domains = lists.openstack.org
local_parts = openstack-discuss
transport = local_copy
unseen
- mailman_router: |
driver = accept
domains = {{ mm_domains }}
local_part_suffix = -admin : \
-bounces : -bounces+* : \
-confirm : -confirm+* : \
-join : -leave : \
-owner : -request : \
-subscribe : -unsubscribe
local_part_suffix_optional
require_files = /var/lib/mailman/core/var/lists/${local_part}.${domain}
transport = mailman_transport
exim_transports:
- local_copy: |
driver = appendfile
file = /var/mail/$local_part
group = mail
mode = 0660
- mailman_transport: |
debug_print = "Email for mailman"
driver = smtp
protocol = lmtp
allow_localhost
hosts = localhost
port = 8024
rcpt_include_affixes = true
- mailman_verp_smtp: |
driver = smtp
headers_add = Errors-To: ${return_path}
headers_remove = Errors-To
max_rcpt = 1
return_path = ${local_part:$return_path}+$local_part=$domain@${domain:$return_path}
mailman_multihost: true

9
inventory/service/groups.yaml
View File

@ -26,11 +26,13 @@ groups:
- kdc03.openstack.org
- eavesdrop01.opendev.org
- paste01.opendev.org
- lists01.opendev.org
# These are test specific hosts that we add to the backup
# group to mimic as much as possible what their prod version
# end up doing.
- gitea99.opendev.org
- review99.opendev.org
- lists99.opendev.org
# All these servers are "special-cased" in specifically
# as they are puppet and should be replaced "soon"
- lists.openstack.org
@ -91,6 +93,7 @@ groups:
- keycloak[0-9]*.opendev.org
- lists.katacontainers.io
- lists.openstack.org
- lists[0-9]*.opendev.org
- meetpad[0-9]*.opendev.org
- mirror[0-9]*.opendev.org
- nb[0-9]*.opendev.org
@ -103,8 +106,10 @@ groups:
- translate[0-9]*.open*.org
- zuul[0-9]*.opendev.org
mailman:
- lists*.katacontainers.io
- lists*.open*.org
- lists.katacontainers.io
- lists.openstack.org
mailman3:
- lists[0-9]*.opendev.org
meetpad:
- meetpad[0-9]*.opendev.org
mirror:

225
inventory/service/host_vars/lists01.opendev.org.yaml Normal file
View File

@ -0,0 +1,225 @@
mm_domains: 'lists.openstack.org:lists.zuul-ci.org:lists.airshipit.org:lists.starlingx.io:lists.opendev.org:lists.openinfra.dev:lists.katacontainers.io'
exim_local_domains: "@:{{ mm_domains }}"
exim_enable_spf: true
exim_aliases:
root: "{{ ','.join(listadmins|default([])) }}"
interop-wg: openstack-discuss
openstack: openstack-discuss
openstack-dev: openstack-discuss
openstack-infra: openstack-discuss
openstack-operators: openstack-discuss
openstack-security: openstack-discuss
openstack-sigs: openstack-discuss
openstack-tc: openstack-discuss
user-committee: openstack-discuss
airship-discuss-owner: spam
community-owner: spam
edge-computing-owner: spam
foundation-board-confidential-owner: spam
foundation-board-owner: spam
foundation-owner: spam
legal-discuss-owner: spam
mailman-owner: spam
marketing-owner: spam
openstack-announce-owner: spam
openstack-docs-owner: spam
openstack-fr-owner: spam
openstack-i18n-owner: spam
openstack-infra-owner: spam
openstack-ko-owner: spam
openstack-qa-owner: spam
product-wg-owner: spam
user-committee-owner: spam
spam: ':fail: delivery temporarily disabled due to ongoing spam flood'
# TODO It would be better to bypass verification for postorius@listdomain
# and set a :fail: rule for anyone trying to send email to this addr.
# But that requires updating our main exim config so that needs more thought.
postorius: ':blackhole: outgoing email only from this address'
exim_domain_aliases:
community@lists.openstack.org: community@lists.openinfra.dev
edge-computing@lists.openstack.org: edge-computing@lists.opendev.org
foundation@lists.openstack.org: foundation@lists.openinfra.dev
foundation-board@lists.openstack.org: foundation-board@lists.openinfra.dev
foundation-board-confidential@lists.openstack.org: foundation-board-confidential@lists.openinfra.dev
goldmembers@lists.openstack.org: goldmembers@lists.openinfra.dev
marketing@lists.openstack.org: marketing@lists.openinfra.dev
staff@lists.openstack.org: staff@lists.openinfra.dev
summit-programming-committee@lists.openinfra.dev: summit-track-chairs@lists.openinfra.dev
summitsponsors@lists.openstack.org: summitsponsors@lists.openinfra.dev
mailman_sites:
# First entry in this list is the primary web domain
- listdomain: lists.opendev.org
install_languages: ['en']
lists:
- name: computing-force-network
description: 'Organizing efforts around Computing Force Network related area'
owner: 'niujie@outlook.com'
- name: edge-computing
description: 'Organizing efforts around the edge-computing focus area.'
owner: 'ildiko@openinfra.dev'
- name: floss-mooc
description: 'Discussions & Coordination around the FLOSS MOOC being collaboratively developed here: https://gitlab.com/mooc-floss/mooc-floss'
owner: 'knelson@openinfra.dev'
- name: nbmp-discuss
description: 'Collaborating on Network Based Media Processing related platform and infrastructure systems usage and development.'
owner: 'ildiko@openstack.org'
- name: openinfralabs
description: 'Discussion of the OpenInfra Labs academic and research resource sharing effort'
owner: 'mnaser@vexxhost.com'
- name: rust-vmm
description: 'Collaborating on Rust-based virtual machine monitors.'
owner: 'claire@openstack.org'
- name: rustyk8s
description: 'Collaborating on Rust-based Kubernetes API.'
owner: 'allison@lohutok.net'
- name: service-announce
description: 'Announcement list for OpenDev services.'
owner: 'cboylan@sapwetik.org'
- name: service-discuss
description: 'Discussion list for OpenDev services.'
owner: 'cboylan@sapwetik.org'
- name: service-incident
description: 'Private list for OpenDev incident coordination.'
owner: 'cboylan@sapwetik.org'
private: true
# The domains and lists below are currently commented out as we intend on
# deploying a single domain and its lists at a time starting with
# lists.opendev.org. As we deploy other domains we can uncomment these
# blocks. Double check no new lists are been added or removed first.
#- listdomain: lists.airshipit.org
# install_languages: ['en']
# lists:
# - name: airship-announce
# description: 'Announcements of Airship releases and other important information.'
# owner: 'jonathan@openstack.org'
# - name: airship-discuss
# description: 'Discussion of Airship usage and development.'
# owner: 'jonathan@openstack.org'
# - name: airship-embargo-notice
# description: 'Embargoed security vulnerability announcements for Airship consumers.'
# owner: 'andrew.walters@att.com'
# private: true
# - name: airship-job-failures
# description: 'Notification messages for failures from CICD jobs.'
# owner: 'roman.gorshunov@att.com'
# - name: airship-security
# description: 'Public Airship security advisories.'
# owner: 'andrew.walters@att.com'
#- listdomain: lists.katacontainers.io
# install_languages: ['en']
# lists:
# - name: embargo-notice
# description: 'Announcements of embargoed notices for the Kata Containers project'
# owner: 'jonathan@openstack.org'
# private: true
# - name: kata-dev
# description: 'Kata Containers Development Mailing List (not for usage questions)'
# owner: 'jonathan@openstack.org'
# - name: kata-hypervisor
# description: 'Discussion of security and virtualization targeted at container use cases'
# owner: 'jonathan@openstack.org'
#- listdomain: lists.openinfra.dev
# install_languages: ['en']
# lists:
# - name: community
# description: 'The OpenInfra Community team is the main contact point for anybody running a local OpenInfra Group.'
# owner: 'allison@openinfra.dev'
# - name: foundation
# description: 'General discussion list for activities of the OpenInfra Foundation'
# owner: 'jonathan@openinfra.dev'
# - name: foundation-board
# description: 'OpenInfra Foundation Board of Directors'
# owner: 'jonathan@openinfra.dev'
# - name: foundation-board-confidential
# description: 'OpenInfra Foundation Board of Directors'
# owner: 'jonathan@openinfra.dev'
# private: true
# - name: goldmembers
# description: 'The discussion list for Gold Members of the OpenInfra Foundation'
# owner: 'jonathan@openinfra.dev'
# private: true
# - name: marketing
# description: 'The OpenInfra Marketing list is the meant to facilitate discussion and best practice sharing among marketers and event organizers in the OpenInfra community.'
# owner: 'allison@openinfra.dev'
# - name: staff
# description: 'Private list for OpenInfra Foundation staff members'
# owner: 'mark@openinfra.dev'
# private: true
# - name: summit-track-chairs
# description: 'OpenInfra Summit track chair communications'
# owner: 'erin@openinfra.dev'
# private: true
# - name: summitsponsors
# description: 'Coordination among OpenInfra Summit event sponsors'
# owner: 'erin@openinfra.dev'
# private: true
#- listdomain: lists.openstack.org
# install_languages: ['de', 'fr', 'it', 'ko', 'ru', 'vi', 'zh_TW']
# lists:
# - name: embargo-notice
# description: 'Announcements to stakeholders for embargoed security vulnerabilities.'
# owner: 'fungi@yuggoth.org'
# private: true
# - name: legal-discuss
# description: 'Discussions on legal matters related to the project'
# owner: 'thierry@openinfra.dev'
# - name: openstack-announce
# description: 'Key announcements about OpenStack & Security advisories'
# owner: 'fungi@yuggoth.org'
# - name: openstack-discuss
# description: 'Discussion of OpenStack usage and development.'
# owner: 'fungi@yuggoth.org'
# - name: openstack-es
# description: 'Lista de correo acerca de OpenStack en español'
# owner: 'flavio@redhat.com'
# - name: openstack-fr
# description: 'List of the OpenStack french user group'
# owner: 'erwan@erwan.com'
# - name: openstack-hpc
# description: 'High-Performance Computing OpenStack List'
# owner: 'brian.schott@nimbisservices.com'
# - name: openstack-i18n
# description: 'List of the OpenStack Internationalization team.'
# owner: 'guoyingc@cn.ibm.com'
# - name: openstack-it
# description: 'Discussioni su OpenStack in italiano'
# owner: 'stefano@openstack.org'
# - name: openstack-ko
# description: 'OpenStack Korea Community Discussions in Korean (오픈스택 한국 커뮤니티 메일링리스트)'
# owner: 'ianyrchoi@gmail.com'
# - name: openstack-mentoring
# description: 'List to coordinate interactions between mentors and mentees of the OpenStack mentoring program. Also for questions about the mentoring program (i.e. how to get involved, how it works, etc.'
# owner: 'amy@demarco.com'
# - name: openstack-stable-maint
# description: 'A mailing list for the OpenStack Stable Branch test reports.'
# owner: 'tony@bakeyournoodle.com'
# - name: openstack-zh
# description: 'OpenStack社区中文讨论群组'
# owner: 'yeluaiesec@gmail.com'
# - name: release-announce
# description: 'Announcement of official OpenStack releases.'
# owner: 'thierry@openstack.org'
# - name: release-job-failures
# description: 'Notification messages for failures from release-related build jobs.'
# owner: 'doug@doughellmann.com'
#- listdomain: lists.starlingx.io
# install_languages: ['en']
# lists:
# - name: starlingx-announce
# description: 'Announcements of StarlingX releases and other important information.'
# owner: 'jonathan@openstack.org'
# - name: starlingx-discuss
# description: 'Discussion of StarlingX usage and development.'
# owner: 'jonathan@openstack.org'
#- listdomain: lists.zuul-ci.org
# install_languages: ['en']
# lists:
# - name: zuul-announce
# description: 'Announcements of Zuul releases and other important information.'
# owner: 'corvus@inaugust.com'
# - name: zuul-discuss
# description: 'Discussion of Zuul usage and development.'
# owner: 'corvus@inaugust.com'
# - name: zuul-jobs-failures
# description: 'Gets notifications about zuul-jobs periodic job failures.'
# owner: 'ssbarnea@redhat.com'

3
playbooks/roles/letsencrypt-create-certs/handlers/main.yaml
View File

@ -45,6 +45,9 @@
- name: letsencrypt updated lists-openstack-org-main
include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
- name: letsencrypt updated lists-opendev-org-main
include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
# Static
- name: letsencrypt updated static-opendev-org-main
include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml

1
playbooks/roles/mailman3/README.rst Normal file
View File

@ -0,0 +1 @@
Role to configure mailman3.

10
playbooks/roles/mailman3/files/99-max_allowed_packet.cnf Normal file
View File

@ -0,0 +1,10 @@
[mysqldump]
# Default is 24MB which is not large enough for all mailman attachments.
# This affects clients including mysqldump. It is larger than the server
# side because mysqldump apparently can do larger packets to insert blobs.
max_allowed_packet=256M
[mysqld]
# Default is 16MB which is not large enough for all mailman attachments.
# This affects the server side.
max_allowed_packet=128M

429
playbooks/roles/mailman3/files/web-settings.py Normal file
View File

@ -0,0 +1,429 @@
# This file has been copied from:
# https://github.com/maxking/docker-mailman/blob/2693386453ff3865b7c106c6aa456b683bd3bf08/web/mailman-web/settings.py
# In order to override the ALLOWED_HOSTS setting.
#
# -*- coding: utf-8 -*-
# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
#
# This file is part of Mailman Suite.
#
# Mailman Suite is free sofware: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# Mailman Suite is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
# You should have received a copy of the GNU General Public License along
# with Mailman Suite. If not, see <http://www.gnu.org/licenses/>.
"""
Django Settings for Mailman Suite (hyperkitty + postorius)
For more information on this file, see
https://docs.djangoproject.com/en/1.8/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.8/ref/settings/
"""
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
import dj_database_url
import sys
from socket import gethostbyname
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.environ.get('SECRET_KEY')
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
ADMINS = (
('Mailman Suite Admin', 'root@localhost'),
)
SITE_ID = 1
# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/3.1/ref/settings/#allowed-hosts
ALLOWED_HOSTS = [
"localhost", # Archiving API from Mailman, keep it.
"127.0.0.1", # Archiving API from Mailman, keep it. OpenDev edit
# "lists.your-domain.org",
# Add here all production URLs you may have.
# The next two entries are commented out to prevent name resolution
# problems. This is an opendev local edit.
# Note we cannot use settings_local.py as this entry is evaluated at
# import time.
#"mailman-web",
#gethostbyname("mailman-web"),
os.environ.get('SERVE_FROM_DOMAIN'),
#os.environ.get('DJANGO_ALLOWED_HOSTS'),
]
# We have modified handling of DJANGO_ALLOWED_HOSTS here to deserialize a
# list of hosts into a python list of strings.
django_allowed_hosts = os.environ.get('DJANGO_ALLOWED_HOSTS')
if django_allowed_hosts:
ALLOWED_HOSTS.extend(django_allowed_hosts.split(':'))
# Mailman API credentials
MAILMAN_REST_API_URL = os.environ.get('MAILMAN_REST_URL', 'http://mailman-core:8001')
MAILMAN_REST_API_USER = os.environ.get('MAILMAN_REST_USER', 'restadmin')
MAILMAN_REST_API_PASS = os.environ.get('MAILMAN_REST_PASSWORD', 'restpass')
MAILMAN_ARCHIVER_KEY = os.environ.get('HYPERKITTY_API_KEY')
MAILMAN_ARCHIVER_FROM = (os.environ.get('MAILMAN_HOST_IP', gethostbyname(os.environ.get('MAILMAN_HOSTNAME', 'mailman-core'))),)
# Application definition
INSTALLED_APPS = []
DEFAULT_APPS = [
'hyperkitty',
'postorius',
'django_mailman3',
# Uncomment the next line to enable the admin:
'django.contrib.admin',
# Uncomment the next line to enable admin documentation:
# 'django.contrib.admindocs',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.sites',
'django.contrib.messages',
'django.contrib.staticfiles',
'rest_framework',
'django_gravatar',
'compressor',
'haystack',
'django_extensions',
'django_q',
'allauth',
'allauth.account',
'allauth.socialaccount',
]
MAILMAN_WEB_SOCIAL_AUTH = [
'django_mailman3.lib.auth.fedora',
'allauth.socialaccount.providers.openid',
'allauth.socialaccount.providers.github',
'allauth.socialaccount.providers.gitlab',
'allauth.socialaccount.providers.google',
]
MIDDLEWARE = (
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.middleware.locale.LocaleMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'django.middleware.security.SecurityMiddleware',
'django_mailman3.middleware.TimezoneMiddleware',
'postorius.middleware.PostoriusMiddleware',
)
ROOT_URLCONF = 'urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.i18n',
'django.template.context_processors.media',
'django.template.context_processors.static',
'django.template.context_processors.tz',
'django.template.context_processors.csrf',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
'django_mailman3.context_processors.common',
'hyperkitty.context_processors.common',
'postorius.context_processors.postorius',
],
},
},
]
WSGI_APPLICATION = 'wsgi.application'
# Database
# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
# dj_database_url uses $DATABASE_URL environment variable to create a
# django-style-config-dict.
# https://github.com/kennethreitz/dj-database-url
DATABASES = {
'default': dj_database_url.config(conn_max_age=600)
}
# If you're behind a proxy, use the X-Forwarded-Host header
# See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host
USE_X_FORWARDED_HOST = True
# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/1.8/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
STATIC_ROOT = '/opt/mailman-web-data/static'
STATIC_URL = '/static/'
# Additional locations of static files
# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
'django.contrib.staticfiles.finders.FileSystemFinder',
'django.contrib.staticfiles.finders.AppDirectoriesFinder',
'compressor.finders.CompressorFinder',
)
SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
LOGIN_URL = 'account_login'
LOGIN_REDIRECT_URL = 'list_index'
LOGOUT_URL = 'account_logout'
# Use SERVE_FROM_DOMAIN as the default domain in the email.
hostname = os.environ.get('SERVE_FROM_DOMAIN', 'localhost.local')
DEFAULT_FROM_EMAIL = 'postorius@{}'.format(hostname)
SERVER_EMAIL = 'root@{}'.format(hostname)
# Change this when you have a real email backend
EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
EMAIL_HOST = os.environ.get('SMTP_HOST', '')
EMAIL_PORT = os.environ.get('SMTP_PORT', 25)
EMAIL_HOST_USER = os.environ.get('SMTP_HOST_USER', '')
EMAIL_HOST_PASSWORD = os.environ.get('SMTP_HOST_PASSWORD', '')
EMAIL_USE_TLS = os.environ.get('SMTP_USE_TLS', False)
EMAIL_USE_SSL = os.environ.get('SMTP_USE_SSL', False)
# Compatibility with Bootstrap 3
from django.contrib.messages import constants as messages # flake8: noqa
MESSAGE_TAGS = {
messages.ERROR: 'danger'
}
#
# Social auth
#
AUTHENTICATION_BACKENDS = (
'django.contrib.auth.backends.ModelBackend',
'allauth.account.auth_backends.AuthenticationBackend',
)
# Django Allauth
ACCOUNT_AUTHENTICATION_METHOD = "username_email"
ACCOUNT_EMAIL_REQUIRED = True
ACCOUNT_EMAIL_VERIFICATION = "mandatory"
# You probably want https in production, but this is a dev setup file
ACCOUNT_DEFAULT_HTTP_PROTOCOL = "https"
ACCOUNT_UNIQUE_EMAIL = True
SOCIALACCOUNT_PROVIDERS = {
'openid': {
'SERVERS': [
dict(id='yahoo',
name='Yahoo',
openid_url='http://me.yahoo.com'),
],
},
'google': {
'SCOPE': ['profile', 'email'],
'AUTH_PARAMS': {'access_type': 'online'},
},
'facebook': {
'METHOD': 'oauth2',
'SCOPE': ['email'],
'FIELDS': [
'email',
'name',
'first_name',
'last_name',
'locale',
'timezone',
],
'VERSION': 'v2.4',
},
}
# django-compressor
# https://pypi.python.org/pypi/django_compressor
#
COMPRESS_PRECOMPILERS = (
('text/less', 'lessc {infile} {outfile}'),
('text/x-scss', 'sassc -t compressed {infile} {outfile}'),
('text/x-sass', 'sassc -t compressed {infile} {outfile}'),
)
# On a production setup, setting COMPRESS_OFFLINE to True will bring a
# significant performance improvement, as CSS files will not need to be
# recompiled on each requests. It means running an additional "compress"
# management command after each code upgrade.
# http://django-compressor.readthedocs.io/en/latest/usage/#offline-compression
# COMPRESS_OFFLINE = True
#
# Full-text search engine
#
HAYSTACK_CONNECTIONS = {
'default': {
'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
'PATH': "/opt/mailman-web-data/fulltext_index",
# You can also use the Xapian engine, it's faster and more accurate,
# but requires another library.
# http://django-haystack.readthedocs.io/en/v2.4.1/installing_search_engines.html#xapian
# Example configuration for Xapian:
#'ENGINE': 'xapian_backend.XapianEngine'
},
}
import sys
# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'filters': {
'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse'
}
},
'handlers': {
'mail_admins': {
'level': 'ERROR',
'filters': ['require_debug_false'],
'class': 'django.utils.log.AdminEmailHandler'
},
'file':{
'level': 'INFO',
'class': 'logging.handlers.RotatingFileHandler',
#'class': 'logging.handlers.WatchedFileHandler',
'filename': os.environ.get('DJANGO_LOG_URL','/opt/mailman-web-data/logs/mailmanweb.log'),
'formatter': 'verbose',
},
'console': {
'class': 'logging.StreamHandler',
'formatter': 'simple',
'level': 'INFO',
'stream': sys.stdout,
},
},
'loggers': {
'django.request': {
'handlers': ['mail_admins', 'file'],
'level': 'INFO',
'propagate': True,
},
'django': {
'handlers': ['file'],
'level': 'INFO',
'propagate': True,
},
'hyperkitty': {
'handlers': ['file'],
'level': 'INFO',
'propagate': True,
},
'postorius': {
'handlers': ['file'],
'level': 'INFO',
'propagate': True
},
},
'formatters': {
'verbose': {
'format': '%(levelname)s %(asctime)s %(process)d %(name)s %(message)s'
},
'simple': {
'format': '%(levelname)s %(message)s'
},
},
}
if os.environ.get('LOG_TO_CONSOLE') == 'yes':
LOGGING['loggers']['django']['handlers'].append('console')
LOGGING['loggers']['django.request']['handlers'].append('console')
# HyperKitty-specific
#
# Only display mailing-lists from the same virtual host as the webserver
FILTER_VHOST = False
Q_CLUSTER = {
'timeout': 300,
'retry': 300,
'save_limit': 100,
'orm': 'default',
}
POSTORIUS_TEMPLATE_BASE_URL = os.environ.get('POSTORIUS_TEMPLATE_BASE_URL', 'http://mailman-web:8000')
DISKCACHE_PATH = os.environ.get('DISKCACHE_PATH', '/opt/mailman-web-data/diskcache')
DISKCACHE_SIZE = os.environ.get('DISKCACHE_SIZE', 2 ** 30) # 1 gigabyte
CACHES = {
'default': {
'BACKEND': 'diskcache.DjangoCache',
'LOCATION': DISKCACHE_PATH,
'OPTIONS': {
'size_limit': DISKCACHE_SIZE,
},
},
}
try:
from settings_local import *
except ImportError:
pass
# Compatibility for older installs that override INSTALLED_APPS
if not INSTALLED_APPS:
INSTALLED_APPS = DEFAULT_APPS + MAILMAN_WEB_SOCIAL_AUTH

21
playbooks/roles/mailman3/files/web-settings_local.py Normal file
View File

@ -0,0 +1,21 @@
# Override default index system. Xapian will be the default in the next
# major release of mailman3, but is currently recommended.
HAYSTACK_CONNECTIONS = {
'default': {
'ENGINE': 'xapian_backend.XapianEngine',
'PATH': "/opt/mailman-web-data/fulltext_index",
},
}
# Disable Gravatar integration since it violates privacy expectations
HYPERKITTY_ENABLE_GRAVATAR = False
# This disables web auth using Google, GitHub, Gitlab, Yahoo,
# Fedora, and generic OpenID.
# TODO: In the future we will want to enable this specifically for
# our keycloak server only.
MAILMAN_WEB_SOCIAL_AUTH = []
FILTER_VHOST = True
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'

9
playbooks/roles/mailman3/handlers/main.yaml Normal file
View File

@ -0,0 +1,9 @@
- name: mailman restart apache2
service:
name: apache2
state: restarted
- name: mailman reload apache2
service:
name: apache2
state: reloaded

114
playbooks/roles/mailman3/tasks/create_lists.yaml Normal file
View File

@ -0,0 +1,114 @@
- name: Check if domain exists
uri:
url: 'http://localhost:8001/3.1/domains/{{ mm_site.listdomain }}'
url_username: restadmin
url_password: "{{ mailman3_rest_password }}"
force_basic_auth: yes
method: GET
body_format: json
status_code: [200, 404]
register: domain_exists
no_log: true
- name: Create list domain in mm3
when: domain_exists.status == 404
uri:
url: 'http://localhost:8001/3.1/domains'
url_username: restadmin
url_password: "{{ mailman3_rest_password }}"
force_basic_auth: yes
method: POST
body_format: json
body:
mail_host: "{{ mm_site.listdomain }}"
status_code: [201]
no_log: true
- name: Check if list exists
uri:
url: 'http://localhost:8001/3.1/lists/{{ mm_list.name }}@{{ mm_site.listdomain }}'
url_username: restadmin
url_password: "{{ mailman3_rest_password }}"
force_basic_auth: yes
method: GET
body_format: json
status_code: [200, 404]
register: list_exists
loop: "{{ mm_site.lists }}"
loop_control:
loop_var: mm_list
no_log: true
- name: Create lists in mm3
when: list_exists.results[exists_idx].status == 404
uri:
url: 'http://localhost:8001/3.1/lists'
url_username: restadmin
url_password: "{{ mailman3_rest_password }}"
force_basic_auth: yes
method: POST
body_format: json
body:
fqdn_listname: "{{ mm_list.name }}@{{ mm_site.listdomain }}"
style_name: "{{ mm_list.private | default('false') | bool | ternary('private-default', 'legacy-default') }}"
status_code: [201]
loop: "{{ mm_site.lists }}"
loop_control:
loop_var: mm_list
index_var: exists_idx
no_log: true
- name: Set list properties in mm3
when: list_exists.results[exists_idx].status == 404
uri:
url: 'http://localhost:8001/3.1/lists/{{ mm_list.name }}@{{ mm_site.listdomain }}/config'
url_username: restadmin
url_password: "{{ mailman3_rest_password }}"
force_basic_auth: yes
method: PATCH
body_format: json
body:
description: "{{ mm_list.description }}"
advertised: "{{ mm_list.private | default('false') | bool | ternary('false', 'true') }}"
# TODO enable this when lynx is present on the container images
# convert_html_to_plaintext: "true"
process_bounces: "false"
filter_extensions:
- "exe"
- "bat"
- "cmd"
- "com"
- "pif"
- "scr"
- "vbs"
- "cpl"
pass_types:
- "multipart/mixed"
- "multipart/alternative"
- "text/plain"
status_code: [204]
loop: "{{ mm_site.lists }}"
loop_control:
loop_var: mm_list
index_var: exists_idx
no_log: true
- name: Set list owner in mm3
when: list_exists.results[exists_idx].status == 404
uri:
url: 'http://localhost:8001/3.1/members'
url_username: restadmin
url_password: "{{ mailman3_rest_password }}"
force_basic_auth: yes
method: POST
body_format: json
body:
list_id: "{{ mm_list.name }}.{{ mm_site.listdomain }}"
subscriber: "{{ mm_list.owner }}"
role: "owner"
status_code: [201]
loop: "{{ mm_site.lists }}"
loop_control:
loop_var: mm_list
index_var: exists_idx
no_log: true

278
playbooks/roles/mailman3/tasks/main.yaml Normal file
View File

@ -0,0 +1,278 @@
# The old mailman2 exim config refers to this file. Write it out
# to make basic testing happy, but we may need to clean it up or
# modify it for mailman3.
- name: Write /etc/aliases.domain
template:
src: "domain_aliases.j2"
dest: "/etc/aliases.domain"
mode: '0444'
- name: Create Mailman Group
group:
name: mailman
gid: 10010
system: yes
- name: Create Mailman User
user:
name: mailman
uid: 10010
comment: Mailman User
shell: /bin/bash
home: /var/lib/mailman
group: mailman
create_home: yes
system: yes
#### Install Mailman ####
- name: Ensure Mailman core volume directory exists
file:
state: directory
path: "/var/lib/mailman/core"
# TODO: undo for https://github.com/maxking/docker-mailman/issues/550
owner: 100
group: 65533
mode: '0755'
- name: Ensure Mailman database volume directory exists
file:
state: directory
path: "/var/lib/mailman/database"
# TODO: undo for https://github.com/maxking/docker-mailman/issues/550
owner: 999
group: 999
mode: '0755'
- name: Ensure Mailman web volume directories exist
file:
state: directory
path: "/var/lib/mailman/{{ item }}"
# TODO: undo for https://github.com/maxking/docker-mailman/issues/550
owner: 100
group: 101
mode: '0755'
loop:
- import
- web
- web-data
- web-data/fulltext_index
- web-data/mm2archives
- name: Copy our overridden settings.py for mailman-web
copy:
src: web-settings.py
dest: /var/lib/mailman/web/settings.py
# TODO: undo for https://github.com/maxking/docker-mailman/issues/550
owner: 100
group: 101
mode: '0644'
- name: Copy our settings_local.py for mailman-web
copy:
src: web-settings_local.py
dest: /var/lib/mailman/web-data/settings_local.py
# TODO: undo for https://github.com/maxking/docker-mailman/issues/550
owner: 100
group: 101
mode: '0644'
- name: Copy our max_allowed_packet override config
copy:
src: 99-max_allowed_packet.cnf
dest: /var/lib/mailman/99-max_allowed_packet.cnf
owner: 999
group: 999
mode: '0644'
- name: Ensure /etc/mailman-compose directory
file:
state: directory
path: /etc/mailman-compose
mode: '0755'
- name: Put docker-compose file in place
template:
src: docker-compose.yaml.j2
dest: /etc/mailman-compose/docker-compose.yaml
mode: '0600'
- name: Run docker-compose pull
shell:
cmd: docker-compose pull
chdir: /etc/mailman-compose/
- name: Run docker-compose up
shell:
cmd: docker-compose up -d
chdir: /etc/mailman-compose/
- name: Run docker prune to cleanup unneeded images
shell:
cmd: docker image prune -f
- name: Install apache2
package:
name:
- apache2
- apache2-utils
state: present
- name: Apache modules
apache2_module:
state: present
name: "{{ a2_mod }}"
loop:
- authz_host
- proxy
- proxy_uwsgi
- ssl
- rewrite
loop_control:
loop_var: a2_mod
notify: mailman restart apache2
- name: Make sure packaged default site disabled
command: a2dissite 000-default.conf
args:
removes: /etc/apache2/sites-enabled/000-default.conf
- name: Create mailman vhost config
template:
src: mailman.vhost.j2
dest: "/etc/apache2/sites-enabled/50-{{ mailman_sites.0.listdomain }}.conf"
owner: root
group: root
mode: '0644'
notify: mailman reload apache2
- name: Enable apache2 server
service:
name: "apache2"
enabled: yes
#### Configure Mailman Services ####
- name: Wait for mm3 REST API to be up and running
uri:
url: 'http://localhost:8001/3.1/domains'
url_username: restadmin
url_password: "{{ mailman3_rest_password }}"
force_basic_auth: yes
method: GET
register: mm_rest_api_up
delay: 1
retries: 300
until: mm_rest_api_up and mm_rest_api_up.status == 200
no_log: true
# It has been difficult to nail down a reliable mathod for determining
# when the database is sufficiently populated that we can create the django
# admin user. We apply a number of approaches in response to this. If we
# can identify a single method that is reliable this list can be trimmed.
- name: Wait for DB to be populated
command: >
docker exec mailman-compose_database_1 bash -c
'mysql -u mailman -p"$MYSQL_PASSWORD" -D mailmandb -e
"SHOW TABLES LIKE \"auth_user\";"'
register: django_db_exists
delay: 1
retries: 300
until: django_db_exists.stdout_lines | length > 1 and django_db_exists.stdout_lines[1] == "auth_user"
- name: Wait for DB to be populated second approach
command: >
docker exec mailman-core alembic -c /usr/lib/python3.9/site-packages/mailman/config/alembic.cfg current
register: alembic_version
delay: 1
retries: 300
until: alembic_version.stdout_lines | length > 0 and "(head)" in alembic_version.stdout_lines[0]
- name: Wait for DB to be populated third approach
shell: >
docker exec mailman-web bash -c
'python3 manage.py showmigrations' |
grep -q '^ \[ \] [0-9]\+_.*'
register: django_db_migrations
delay: 1
retries: 300
failed_when: false
# When grep stops matching the empty '[ ]' that indicates all migrations
# are marked with '[X]' and are complete. Grep returns non zero when we
# reach this point.
until: django_db_migrations.rc != 0
- name: Check if django admin user exists
command: >
docker exec mailman-compose_database_1 bash -c
'mysql -u mailman -p"$MYSQL_PASSWORD" -D mailmandb -e
"SELECT COUNT(id) FROM auth_user WHERE id = 1 AND is_superuser = 1;"'
register: django_admin_exists
- name: Create django admin user
when: django_admin_exists.stdout_lines[1] == "0"
command: >
docker exec mailman-web bash -c
"DJANGO_SUPERUSER_PASSWORD={{ mailman3_admin_password }}
python3 manage.py createsuperuser --no-input
--username {{ mailman3_admin_user }}
--email '{{ mailman3_admin_email }}'"
no_log: true
- name: Create lists in mm3
include_tasks: create_lists.yaml
loop: "{{ mailman_sites }}"
loop_control:
loop_var: mm_site
#### Logrotate for service logs ####
- name: Rotate mailman logs
include_role:
name: logrotate
vars:
logrotate_rotate: 90
logrotate_file_name: '/var/lib/mailman/web-data/logs/*.log'
#### Database Backups ####
- name: Create db backup dest
file:
state: directory
path: /var/backups/mailman-mariadb
mode: 0700
owner: root
group: root
- name: Set up cron job to backup the database
cron:
name: mailman-db-backup
state: present
user: root
job: >
/usr/local/bin/docker-compose -f /etc/mailman-compose/docker-compose.yaml exec -T database
bash -c '/usr/bin/mysqldump --opt --databases mailmandb --single-transaction -uroot -p"$MYSQL_ROOT_PASSWORD"' |
gzip -9 > /var/backups/mailman-mariadb/mailman-mariadb.sql.gz
minute: 14
hour: 5
- name: Rotate db backups
include_role:
name: logrotate
vars:
logrotate_file_name: /var/backups/mailman-mariadb/mailman-mariadb.sql.gz
logrotate_compress: false
- name: Setup db backup streaming job
block:
- name: Create backup streaming config dir
file:
path: /etc/borg-streams
state: directory
- name: Create db streaming file
copy:
content: >-
/usr/local/bin/docker-compose -f /etc/mailman-compose/docker-compose.yaml exec -T mariadb
bash -c '/usr/bin/mysqldump --skip-extended-insert --databases mailmandb --single-transaction -uroot -p"$MYSQL_ROOT_PASSWORD"'
dest: /etc/borg-streams/mysql

71
playbooks/roles/mailman3/templates/docker-compose.yaml.j2 Normal file
View File

@ -0,0 +1,71 @@
# Adapted from https://github.com/maxking/docker-mailman/blob/2693386453ff3865b7c106c6aa456b683bd3bf08/docker-compose-mysql.yaml
# which is an MIT licensed repo.
version: '2'
services:
mailman-core:
image: docker.io/maxking/mailman-core:0.4
restart: always
container_name: mailman-core
volumes:
- /var/lib/mailman/core:/opt/mailman/
- /var/lib/mailman/import:/opt/import
stop_grace_period: 30s
depends_on:
- database
environment:
- DATABASE_URL=mysql+pymysql://mailman:{{ mailman3_db_password }}@127.0.0.1:3306/mailmandb?charset=utf8mb4&use_unicode=1
- DATABASE_TYPE=mysql
- DATABASE_CLASS=mailman.database.mysql.MySQLDatabase
- HYPERKITTY_URL=http://127.0.0.1:8000/hyperkitty
- HYPERKITTY_API_KEY={{ mailman3_hyperkitty_api_key }}
- SMTP_HOST=localhost
- MM_HOSTNAME=localhost
- MAILMAN_REST_USER=restadmin
- MAILMAN_REST_PASSWORD={{ mailman3_rest_password }}
network_mode: host
#user: mailman
mailman-web:
image: docker.io/maxking/mailman-web:0.4
restart: always
container_name: mailman-web
depends_on:
- database
volumes:
- /var/lib/mailman/import:/opt/import
- /var/lib/mailman/web-data:/opt/mailman-web-data
- /var/lib/mailman/web/settings.py:/opt/mailman-web/settings.py
environment:
# Testing to see if these are really necessary
#- MAILMAN_ADMIN_USER={{ mailman3_admin_user }}
#- MAILMAN_ADMIN_EMAIL={{ mailman3_admin_email }}
- SERVE_FROM_DOMAIN=lists.opendev.org
- DJANGO_ALLOWED_HOSTS={{ mm_domains }}
- DATABASE_TYPE=mysql
- DATABASE_URL=mysql://mailman:{{ mailman3_db_password }}@127.0.0.1:3306/mailmandb?charset=utf8mb4
- HYPERKITTY_API_KEY={{ mailman3_hyperkitty_api_key }}
- SECRET_KEY={{ mailman3_django_secret_key }}
- DYLD_LIBRARY_PATH=/usr/local/mysql/lib/
- MAILMAN_HOSTNAME=localhost
- MAILMAN_REST_URL=http://127.0.0.1:8001
- MAILMAN_REST_USER=restadmin
- MAILMAN_REST_PASSWORD={{ mailman3_rest_password }}
- POSTORIUS_TEMPLATE_BASE_URL=http://127.0.0.1:8000
- SMTP_HOST=localhost
network_mode: host
#user: mailman
database:
environment:
MYSQL_DATABASE: mailmandb
MYSQL_USER: mailman
MYSQL_PASSWORD: {{ mailman3_db_password }}
MYSQL_ROOT_PASSWORD: {{ mailman3_db_root_password }}
image: docker.io/library/mariadb:10.6
restart: always
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
volumes:
- /var/lib/mailman/database:/var/lib/mysql
- /var/lib/mailman/99-max_allowed_packet.cnf:/etc/mysql/conf.d/99-max_allowed_packet.cnf:ro
network_mode: host

6
playbooks/roles/mailman3/templates/domain_aliases.j2 Normal file
View File

@ -0,0 +1,6 @@
# /etc/aliases.domain
{% for k, v in exim_domain_aliases|dictsort %}
{% if v %}
{{ k }}: {{ v }}
{% endif %}
{% endfor %}

69
playbooks/roles/mailman3/templates/mailman.vhost.j2 Normal file
View File

@ -0,0 +1,69 @@
<VirtualHost *:80>
ServerName {{ mailman_sites.0.listdomain }}
{% for site in mailman_sites[1:] -%}
ServerAlias {{ site.listdomain }}
{% endfor -%}
ErrorLog ${APACHE_LOG_DIR}/{{ mailman_sites.0.listdomain }}-error.log
# Possible values include: debug, info, notice, warn, error, crit,
# alert, emerg.
LogLevel warn
CustomLog ${APACHE_LOG_DIR}/{{ mailman_sites.0.listdomain }}-access.log combined
# Use mod rewrite to redirect as we want to preserve the FQDN for each
# mm3 vhost.
RewriteEngine On
RewriteRule "/(.*)" "https://%{HTTP_HOST}/$1" [R=301]
</VirtualHost>
<VirtualHost *:443>
ServerName {{ mailman_sites.0.listdomain }}
{% for site in mailman_sites[1:] -%}
ServerAlias {{ site.listdomain }}
{% endfor -%}
ServerAdmin webmaster@openstack.org
ErrorLog ${APACHE_LOG_DIR}/{{ mailman_sites.0.listdomain }}-ssl-error.log
LogLevel warn
CustomLog ${APACHE_LOG_DIR}/{{ mailman_sites.0.listdomain }}-ssl-access.log combined
SSLEngine on
SSLProtocol All -SSLv2 -SSLv3
# Note: this list should ensure ciphers that provide forward secrecy
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
SSLHonorCipherOrder on
SSLCertificateFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.cer
SSLCertificateKeyFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.key
SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer
Alias /static /var/lib/mailman/web-data/static
Alias /favicon.ico /var/lib/mailman/web-data/static/hyperkitty/img/favicon.ico
<Location "/admin">
Require local
</Location>
RewriteEngine On
RewriteRule "/pipermail/(.*)" "/var/lib/mailman/web-data/mm2archives/%{HTTP_HOST}/public/$1"
RewriteRule "/cgi-bin/mailman/listinfo/(.*)" "https://%{HTTP_HOST}/postorius/lists/$1.%{HTTP_HOST}/"
RewriteRule "/cgi-bin/mailman/listinfo" "https://%{HTTP_HOST}/postorius/lists/"
ProxyPassMatch ^/static/ !
ProxyPass "/" "uwsgi://localhost:8080/"
<Directory /var/lib/mailman/web-data/static/>
AllowOverride None
Order allow,deny
Allow from all
Require all granted
</Directory>
<Directory /var/lib/mailman/web-data/mm2archives/>
AllowOverride None
Order allow,deny
Allow from all
Require all granted
</Directory>
</VirtualHost>

10
playbooks/service-lists3.yaml Normal file
View File

@ -0,0 +1,10 @@
# Maintaing a todo list here as a central accounting file
# TODO Test mailman dmarc settings
# TODO fix container uid/gid mismatch with bind mounted contents
# this breaks xapian
- hosts: "mailman3:!disabled"
name: "Configure mailman3 servers"
roles:
- iptables
- install-docker
- mailman3

294
playbooks/zuul/files/host_vars/lists99.opendev.org.yaml Normal file
View File

@ -0,0 +1,294 @@
mailman_list_password: notarealpassword
mailman3_db_password: Eith5vii5beezohc
mailman3_db_root_password: eiloh9Edohngaeri
mailman3_hyperkitty_api_key: Thosai4Xomeque9e
mailman3_django_secret_key: ohki3ohWusai8tee
mailman3_rest_password: OhTo3doh5ohsuope
mailman3_admin_user: admin
mailman3_admin_email: infra-root@openstack.org
mailman3_admin_password: AeNie8vegeiquei1
mm_domains: 'lists.openstack.org:lists.zuul-ci.org:lists.airshipit.org:lists.starlingx.io:lists.opendev.org:lists.openinfra.dev:lists.katacontainers.io'
exim_local_domains: "@:{{ mm_domains }}"
exim_enable_spf: true
exim_aliases:
root: "{{ ','.join(listadmins|default([])) }}"
interop-wg: openstack-discuss
openstack: openstack-discuss
openstack-dev: openstack-discuss
openstack-infra: openstack-discuss
openstack-operators: openstack-discuss
openstack-security: openstack-discuss
openstack-sigs: openstack-discuss
openstack-tc: openstack-discuss
user-committee: openstack-discuss
airship-discuss-owner: spam
community-owner: spam
edge-computing-owner: spam
foundation-board-confidential-owner: spam
foundation-board-owner: spam
foundation-owner: spam
legal-discuss-owner: spam
mailman-owner: spam
marketing-owner: spam
openstack-announce-owner: spam
openstack-docs-owner: spam
openstack-fr-owner: spam
openstack-i18n-owner: spam
openstack-infra-owner: spam
openstack-ko-owner: spam
openstack-qa-owner: spam
product-wg-owner: spam
user-committee-owner: spam
spam: ':fail: delivery temporarily disabled due to ongoing spam flood'
# TODO It would be better to bypass verification for postorius@listdomain
# and set a :fail: rule for anyone trying to send email to this addr.
# But that requires updating our main exim config so that needs more thought.
postorius: ':blackhole: outgoing email only from this address'
exim_domain_aliases:
community@lists.openstack.org: community@lists.openinfra.dev
edge-computing@lists.openstack.org: edge-computing@lists.opendev.org
foundation@lists.openstack.org: foundation@lists.openinfra.dev
foundation-board@lists.openstack.org: foundation-board@lists.openinfra.dev
foundation-board-confidential@lists.openstack.org: foundation-board-confidential@lists.openinfra.dev
goldmembers@lists.openstack.org: goldmembers@lists.openinfra.dev
marketing@lists.openstack.org: marketing@lists.openinfra.dev
staff@lists.openstack.org: staff@lists.openinfra.dev
summit-programming-committee@lists.openinfra.dev: summit-track-chairs@lists.openinfra.dev
summitsponsors@lists.openstack.org: summitsponsors@lists.openinfra.dev
exim_routers:
- mailman_verp_router: |
{% raw -%}
driver = dnslookup
condition = ${if or{{eq{$sender_host_address}{127.0.0.1}}\
{eq{$sender_host_address}{::1}}}{yes}{no}}
{% endraw %}
domains = !+local_domains
ignore_target_hosts = <; 0.0.0.0; \
64.94.110.11; \
127.0.0.0/8; \
::1/128;fe80::/10;fe \
c0::/10;ff00::/8
senders = "*-bounces@*"
transport = mailman_verp_smtp
- dnslookup: '{{ exim_dnslookup_router }}'
- system_aliases: '{{ exim_system_aliases_router }}'
- domain_aliases: |
driver = redirect
allow_fail
allow_defer
data = ${lookup{$local_part@$domain}lsearch{/etc/aliases.domain}}
file_transport = address_file
pipe_transport = address_pipe
- localuser: '{{ exim_localuser_router }}'
- mailman_copy: |
driver = accept
domains = lists.openstack.org
local_parts = openstack-discuss
transport = local_copy
unseen
- mailman_router: |
driver = accept
domains = {{ mm_domains }}
local_part_suffix = -admin : \
-bounces : -bounces+* : \
-confirm : -confirm+* : \
-join : -leave : \
-owner : -request : \
-subscribe : -unsubscribe
local_part_suffix_optional
require_files = /var/lib/mailman/core/var/lists/${local_part}.${domain}
transport = mailman_transport
exim_transports:
- local_copy: |
driver = appendfile
file = /var/mail/$local_part
group = mail
mode = 0660
- mailman_transport: |
debug_print = "Email for mailman"
driver = smtp
protocol = lmtp
allow_localhost
hosts = localhost
port = 8024
rcpt_include_affixes = true
- mailman_verp_smtp: |
driver = smtp
headers_add = Errors-To: ${return_path}
headers_remove = Errors-To
max_rcpt = 1
return_path = ${local_part:$return_path}+$local_part=$domain@${domain:$return_path}
mailman_multihost: true
mailman_sites:
# First entry in this list is the primary web domain
- listdomain: lists.opendev.org
install_languages: ['en']
lists:
- name: computing-force-network
description: 'Organizing efforts around Computing Force Network related area'
owner: 'niujie@outlook.com'
- name: edge-computing
description: 'Organizing efforts around the edge-computing focus area.'
owner: 'ildiko@openinfra.dev'
- name: floss-mooc
description: 'Discussions & Coordination around the FLOSS MOOC being collaboratively developed here: https://gitlab.com/mooc-floss/mooc-floss'
owner: 'knelson@openinfra.dev'
- name: nbmp-discuss
description: 'Collaborating on Network Based Media Processing related platform and infrastructure systems usage and development.'
owner: 'ildiko@openstack.org'
- name: openinfralabs
description: 'Discussion of the OpenInfra Labs academic and research resource sharing effort'
owner: 'mnaser@vexxhost.com'
- name: rust-vmm
description: 'Collaborating on Rust-based virtual machine monitors.'
owner: 'claire@openstack.org'
- name: rustyk8s
description: 'Collaborating on Rust-based Kubernetes API.'
owner: 'allison@lohutok.net'
- name: service-announce
description: 'Announcement list for OpenDev services.'
owner: 'cboylan@sapwetik.org'
- name: service-discuss
description: 'Discussion list for OpenDev services.'
owner: 'cboylan@sapwetik.org'
- name: service-incident
description: 'Private list for OpenDev incident coordination.'
owner: 'cboylan@sapwetik.org'
private: true
- listdomain: lists.airshipit.org
install_languages: ['en']
lists:
- name: airship-announce
description: 'Announcements of Airship releases and other important information.'
owner: 'jonathan@openstack.org'
- name: airship-discuss
description: 'Discussion of Airship usage and development.'
owner: 'jonathan@openstack.org'
- name: airship-embargo-notice
description: 'Embargoed security vulnerability announcements for Airship consumers.'
owner: 'andrew.walters@att.com'
private: true
- name: airship-job-failures
description: 'Notification messages for failures from CICD jobs.'
owner: 'roman.gorshunov@att.com'
- name: airship-security
description: 'Public Airship security advisories.'
owner: 'andrew.walters@att.com'
- listdomain: lists.katacontainers.io
install_languages: ['en']
lists:
- name: embargo-notice
description: 'Announcements of embargoed notices for the Kata Containers project'
owner: 'jonathan@openstack.org'
private: true
- name: kata-dev
description: 'Kata Containers Development Mailing List (not for usage questions)'
owner: 'jonathan@openstack.org'
- name: kata-hypervisor
description: 'Discussion of security and virtualization targeted at container use cases'
owner: 'jonathan@openstack.org'
- listdomain: lists.openinfra.dev
install_languages: ['en']
lists:
- name: community
description: 'The OpenInfra Community team is the main contact point for anybody running a local OpenInfra Group.'
owner: 'allison@openinfra.dev'
- name: foundation
description: 'General discussion list for activities of the OpenInfra Foundation'
owner: 'jonathan@openinfra.dev'
- name: foundation-board
description: 'OpenInfra Foundation Board of Directors'
owner: 'jonathan@openinfra.dev'
- name: foundation-board-confidential
description: 'OpenInfra Foundation Board of Directors'
owner: 'jonathan@openinfra.dev'
private: true
- name: goldmembers
description: 'The discussion list for Gold Members of the OpenInfra Foundation'
owner: 'jonathan@openinfra.dev'
private: true
- name: marketing
description: 'The OpenInfra Marketing list is the meant to facilitate discussion and best practice sharing among marketers and event organizers in the OpenInfra community.'
owner: 'allison@openinfra.dev'
- name: staff
description: 'Private list for OpenInfra Foundation staff members'
owner: 'mark@openinfra.dev'
private: true
- name: summit-track-chairs
description: 'OpenInfra Summit track chair communications'
owner: 'erin@openinfra.dev'
private: true
- name: summitsponsors
description: 'Coordination among OpenInfra Summit event sponsors'
owner: 'erin@openinfra.dev'
private: true
- listdomain: lists.openstack.org
install_languages: ['de', 'fr', 'it', 'ko', 'ru', 'vi', 'zh_TW']
lists:
- name: embargo-notice
description: 'Announcements to stakeholders for embargoed security vulnerabilities.'
owner: 'fungi@yuggoth.org'
private: true
- name: legal-discuss
description: 'Discussions on legal matters related to the project'
owner: 'thierry@openinfra.dev'
- name: openstack-announce
description: 'Key announcements about OpenStack & Security advisories'
owner: 'fungi@yuggoth.org'
- name: openstack-discuss
description: 'Discussion of OpenStack usage and development.'
owner: 'fungi@yuggoth.org'
- name: openstack-es
description: 'Lista de correo acerca de OpenStack en español'
owner: 'flavio@redhat.com'
- name: openstack-fr
description: 'List of the OpenStack french user group'
owner: 'erwan@erwan.com'
- name: openstack-hpc
description: 'High-Performance Computing OpenStack List'
owner: 'brian.schott@nimbisservices.com'
- name: openstack-i18n
description: 'List of the OpenStack Internationalization team.'
owner: 'guoyingc@cn.ibm.com'
- name: openstack-it
description: 'Discussioni su OpenStack in italiano'
owner: 'stefano@openstack.org'
- name: openstack-ko
description: 'OpenStack Korea Community Discussions in Korean (오픈스택 한국 커뮤니티 메일링리스트)'
owner: 'ianyrchoi@gmail.com'
- name: openstack-mentoring
description: 'List to coordinate interactions between mentors and mentees of the OpenStack mentoring program. Also for questions about the mentoring program (i.e. how to get involved, how it works, etc.'
owner: 'amy@demarco.com'
- name: openstack-stable-maint
description: 'A mailing list for the OpenStack Stable Branch test reports.'
owner: 'tony@bakeyournoodle.com'
- name: openstack-zh
description: 'OpenStack社区中文讨论群组'
owner: 'yeluaiesec@gmail.com'
- name: release-announce
description: 'Announcement of official OpenStack releases.'
owner: 'thierry@openstack.org'
- name: release-job-failures
description: 'Notification messages for failures from release-related build jobs.'
owner: 'doug@doughellmann.com'
- listdomain: lists.starlingx.io
install_languages: ['en']
lists:
- name: starlingx-announce
description: 'Announcements of StarlingX releases and other important information.'
owner: 'jonathan@openstack.org'
- name: starlingx-discuss
description: 'Discussion of StarlingX usage and development.'
owner: 'jonathan@openstack.org'
- listdomain: lists.zuul-ci.org
install_languages: ['en']
lists:
- name: zuul-announce
description: 'Announcements of Zuul releases and other important information.'
owner: 'corvus@inaugust.com'
- name: zuul-discuss
description: 'Discussion of Zuul usage and development.'
owner: 'corvus@inaugust.com'
- name: zuul-jobs-failures
description: 'Gets notifications about zuul-jobs periodic job failures.'
owner: 'ssbarnea@redhat.com'

16
playbooks/zuul/lists3-alias-logs.yaml Normal file
View File

@ -0,0 +1,16 @@
# We need to do this in a run playbook so that we copy logs even
# when the run playbooks fail. Failure in the run playbooks prevents the
# test playbook from running.
- hosts: "mailman3"
tasks:
- name: Alias mailman-web logs
file:
state: link
src: /var/lib/mailman/web-data/logs
path: /var/lib/mailman/mailman-web-logs
- name: Alias mailman-core logs
file:
state: link
src: /var/lib/mailman/core/var/logs
path: /var/lib/mailman/mailman-core-logs

8
playbooks/zuul/run-base.yaml
View File

@ -151,6 +151,14 @@
- host_vars/paste99.opendev.org.yaml
- host_vars/refstack01.openstack.org.yaml
- host_vars/review99.opendev.org.yaml
- name: Write lists99 host_vars.
# This file is special because it has raw tags in it that we need to
# carry through. I can't figure out a better way to do that then copying
# it directly rather than treating it as a template.
copy:
src: "files/host_vars/lists99.opendev.org.yaml"
dest: "/etc/ansible/hosts/host_vars/lists99.opendev.org.yaml"
- name: Display group membership
command: ansible localhost -m debug -a 'var=groups'
- name: Run base.yaml

19
playbooks/zuul/test-lists3.yaml Normal file
View File

@ -0,0 +1,19 @@
- hosts: "mailman3"
tasks:
- name: Force hyperkitty to populate info about the new lists we created
command: docker exec mailman-web ./manage.py runjobs hourly
- name: Update /etc/hosts in order to test mm3 vhosts
lineinfile:
state: present
backrefs: yes
path: /etc/hosts
regexp: '^127\.0\.0\.1\s+(.*)$'
line: '127.0.0.1 \1 {{ mm_site.listdomain }}'
loop: "{{ mailman_sites }}"
loop_control:
loop_var: mm_site
- name: Run selenium container
include_role:
name: run-selenium

61
testinfra/test_lists_opendev_org.py Normal file
View File

@ -0,0 +1,61 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from util import take_screenshots
testinfra_hosts = ['lists99.opendev.org']
def test_mariadb_listening(host):
mariadb_port = host.socket("tcp://127.0.0.1:3306")
assert mariadb_port.is_listening
def test_mailman3_web_listening(host):
mailman_web = host.socket("tcp://0.0.0.0:8000")
assert mailman_web.is_listening
mailman_uwsgi = host.socket("tcp://0.0.0.0:8080")
assert mailman_uwsgi.is_listening
def test_mailman3_core_listening(host):
mailman_rest = host.socket("tcp://127.0.0.1:8001")
assert mailman_rest.is_listening
mailman_lmtp = host.socket("tcp://127.0.0.1:8024")
assert mailman_lmtp.is_listening
def test_apache2_listening(host):
apache2_http = host.socket("tcp://0.0.0.0:80")
assert apache2_http.is_listening
apache2_https = host.socket("tcp://0.0.0.0:443")
assert apache2_https.is_listening
def test_mailman3_screenshots(host):
shots = (
("https://lists.opendev.org:443", None, "mm3-opendev-main.png"),
("https://lists.opendev.org:443/hyperkitty/",
None, "mm3-opendev-archives.png"),
("https://lists.opendev.org:443"
"/hyperkitty/list/service-discuss@lists.opendev.org/",
None, "mm3-opendev-list.png"),
("https://lists.opendev.org:443"
"/accounts/login/?next=/postorius/lists/",
None, "mm3-opendev-login.png"),
("https://lists.openstack.org:443", None, "mm3-openstack-main.png"),
("https://lists.openstack.org:443/hyperkitty/",
None, "mm3-openstack-archives.png"),
("https://lists.openstack.org:443"
"/hyperkitty/list/openstack-discuss@lists.openstack.org/",
None, "mm3-openstack-list.png"),
("https://lists.openstack.org:443"
"/accounts/login/?next=/postorius/lists/",
None, "mm3-openstack-login.png"),
)
take_screenshots(host, shots)

28
tools/mm3-migrate.sh Normal file
View File

@ -0,0 +1,28 @@
#!/bin/bash
s=$1
test -z "$s" && echo 'Specify the list FQDN to migrate!' && exit 1
echo "*** starting at $(date -Is) ***"
set -x
for m in $(
ls -d /var/lib/mailman/import/$s/lists/* \
| grep -v /mailman$ \
| cut -d/ -f8
); do
time sudo docker-compose -f /etc/mailman-compose/docker-compose.yaml \
exec -T -u mailman mailman-core mailman import21 $m@$s \
/opt/import/$s/lists/$m/config.pck
time sudo docker-compose -f /etc/mailman-compose/docker-compose.yaml \
exec -T -u mailman mailman-web python3 manage.py hyperkitty_import -l \
$m@$s /opt/import/$s/archives/private/$m.mbox/$m.mbox
time sudo docker-compose -f /etc/mailman-compose/docker-compose.yaml \
exec -T -u mailman mailman-web python3 manage.py \
update_index_one_list $m@$s
done
sudo mv /var/lib/mailman/import/$s/archives \
/var/lib/mailman/web-data/mm2archives/$s
for a in /var/lib/mailman/web-data/mm2archives/$s/public/*; do
sudo ln -fs ../private/`basename $a` $a
done
set +x
echo "*** completed at $(date -Is) ***"

16
zuul.d/infra-prod.yaml
View File

@ -571,9 +571,23 @@
- inventory/service/host_vars/lists.katacontainers.io.yaml
- playbooks/roles/iptables/
- playbooks/roles/base/exim
- playbooks/roles/mailman
- playbooks/roles/mailman/
- playbooks/service-lists.yaml
- job:
name: infra-prod-service-lists3
parent: infra-prod-service-base
description: Run service-lists3.yaml playbook.
vars:
playbook_name: service-lists3.yaml
files:
- inventory/base
- inventory/service/host_vars/lists01.opendev.org.yaml
- playbooks/roles/iptables/
- playbooks/roles/base/exim
- playbooks/roles/mailman3/
- playbooks/service-lists3.yaml
# Run AFS changes separately so we can make sure to only do one at a time
# (turns out quorum is nice to have)
- job:

9
zuul.d/project.yaml
View File

@ -30,6 +30,7 @@
soft: true
- system-config-run-kerberos
- system-config-run-lists
- system-config-run-lists3
- system-config-run-nodepool:
dependencies:
- name: opendev-buildset-registry
@ -181,6 +182,7 @@
soft: true
- system-config-run-kerberos
- system-config-run-lists
- system-config-run-lists3
- system-config-run-nodepool:
dependencies:
- name: opendev-buildset-registry
@ -463,6 +465,12 @@
soft: true
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-lists3: &infra-prod-service-lists3
dependencies:
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-mirror: &infra-prod-service-mirror
dependencies:
- name: infra-prod-letsencrypt
@ -605,6 +613,7 @@
- infra-prod-service-keycloak: *infra-prod-service-keycloak
- infra-prod-service-meetpad: *infra-prod-service-meetpad
- infra-prod-service-lists: *infra-prod-service-lists
- infra-prod-service-lists3: *infra-prod-service-lists3
- infra-prod-service-mirror: *infra-prod-service-mirror
- infra-prod-service-nodepool: *infra-prod-service-nodepool
- infra-prod-service-static: *infra-prod-service-static

45
zuul.d/system-config-run.yaml
View File

@ -276,7 +276,7 @@
- inventory/service/host_vars/lists.katacontainers.io.yaml
- inventory/service/group_vars/mailman.yaml
- playbooks/roles/base/exim
- playbooks/roles/mailman
- playbooks/roles/mailman/
- playbooks/service-lists.yaml
- playbooks/test-lists.yaml
- playbooks/zuul/templates/host_vars/lists.openstack.org.yaml.j2
@ -304,6 +304,49 @@
'/var/log/apache2': logs
'/var/log/mailman': logs
- job:
name: system-config-run-lists3
# We don't use the system-config-run-containers base job because we
# are consuming upstream containers only.
parent: system-config-run
description: |
Run the playbook for a mailman3 list server.
timeout: 3600
nodeset:
nodes:
- <<: *bridge_node_x86
- name: lists99.opendev.org
label: ubuntu-jammy
groups:
- <<: *bastion_group
required-projects:
- opendev/system-config
files:
- playbooks/bootstrap-bridge.yaml
- inventory/service/host_vars/lists01.opendev.org.yaml
- inventory/service/group_vars/mailman3.yaml
- playbooks/roles/base/exim
- playbooks/roles/mailman3
- playbooks/service-lists3.yaml
- playbooks/test-lists3.yaml
- playbooks/zuul/files/host_vars/lists99.opendev.org.yaml
- testinfra/test_lists_opendev_org.py
vars:
run_playbooks:
- playbooks/letsencrypt.yaml
- playbooks/service-lists3.yaml
# Run this twice to check idempotency
- playbooks/service-lists3.yaml
- playbooks/zuul/lists3-alias-logs.yaml
run_test_playbook: playbooks/zuul/test-lists3.yaml
host-vars:
lists99.opendev.org:
host_copy_output:
'/var/log/acme.sh': logs
'/var/log/apache2': logs
'/var/lib/mailman/mailman-web-logs': logs
'/var/lib/mailman/mailman-core-logs': logs
- job:
name: system-config-run-nodepool
parent: system-config-run