limnoria/meetbot setup on eavesdrop01.opendev.org

This installs our Limnoira/meetbot container and configures it on
eavesdrop01.opendev.org.  I have ported the configuration from the old
puppet as best I can (it is very verbose); my procedure was to use the
Limnoira wizard to start a new config file then backport everything
from the old file.  I felt this was best to not miss any new options.

This does channel logging (via built-in ChannelLogger plugin, along
with a cron job for logs2html) and runs our fork of meetbot.

It exports the channel logs via HTTP to /irclogs and meetings logs to
/meetings.  meetings.opendev.org will proxy to these two locations
when the server is active.

Note this has not ported the channel list; so the bot will not be
listening in our channels.

Change-Id: I9f9a466c271e1a706f9f98f816de0e84047519f1

inventory/service/groups.yaml

@@ -84,6 +84,7 @@ groups:
     - kdc04.openstack.org
   letsencrypt:
     - codesearch[0-9]*.opendev.org
+    - eavesdrop[0-9]*.opendev.org
     - etherpad[0-9]*.opendev.org
     - ethercalc[0-9]*.open*.org
     - gitea[0-9]*.opendev.org

inventory/service/host_vars/eavesdrop01.opendev.org.yaml

@@ -0,0 +1,3 @@
+letsencrypt_certs:
+  eavesdrop01-opendev-org-main:
+    - eavesdrop01.opendev.org

playbooks/roles/letsencrypt-create-certs/handlers/main.yaml

@@ -11,6 +11,9 @@
 # ("include_tasks" is okay).
 # https://docs.ansible.com/ansible/latest/porting_guides/porting_guide_2.8.html#imports-as-handlers
 
+- name: letsencrypt updated eavesdrop01-opendev-org-main
+  include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+
 - name: letsencrypt updated graphite02-main
   include_tasks: roles/letsencrypt-create-certs/handlers/restart_graphite.yaml
 

playbooks/roles/limnoria/README.rst

@@ -0,0 +1,6 @@
+Setup limnoira and meetbot logging
+
+TODO
+
+ * ubuntu-bots bug tracker to highlight bug links
+   * https://git.launchpad.net/~krytarik/ubuntu-bots/+git/ubuntu-bots/

playbooks/roles/limnoria/defaults/main.yaml

@@ -0,0 +1,176 @@
+limnoria_default_nick: opendevtest
+limnoria_default_nickserv_password: 'abc123'
+limnoria_default_user: 'Opendev IRC Services'
+limnoria_directories_conf: /var/lib/limnoria/opendev/conf
+limnoria_directories_data: /var/lib/limnoria/opendev/data
+limnoria_directories_data_tmp: /var/lib/limnoria/opendev/data/tmp
+limnoria_directories_data_web: /var/lib/limnoria/opendev/data/web
+limnoria_directories_log: /var/lib/limnoria/opendev/logs
+
+limnoria_network_config:
+  oftc:
+    password: ''
+    servers: irc4.oftc.net:6697
+    channels: '#opendev-sandbox'
+    ssl: 'True'
+
+# AVAILABLE SUPYBOT CONFIG KEYS:
+
+# ###
+# # Determines what networks the bot will connect to.
+# #
+# # Default value:  
+# ###
+# supybot.networks: {{ limnoria_networks }}
+
+# ###
+# # Determines what certificate file (if any) the bot will use to connect
+# # with SSL sockets to oftc.
+# #
+# # Default value: 
+# ###
+# supybot.networks.oftc.certfile: 
+
+# ###
+# # Space-separated list of channels the bot will join only on oftc.
+# #
+# # Default value:  
+# ###
+# supybot.networks.oftc.channels: #opendev
+
+# ###
+# # Determines what key (if any) will be used to join the channel.
+# #
+# # Default value: 
+# ###
+# supybot.networks.oftc.channels.key: 
+
+# ###
+# # Determines the bot's ident string, if the server doesn't provide one
+# # by default. If empty, defaults to supybot.ident.
+# #
+# # Default value: 
+# ###
+# supybot.networks.oftc.ident: 
+
+# ###
+# # Determines what nick the bot will use on this network. If empty,
+# # defaults to supybot.nick.
+# #
+# # Default value: 
+# ###
+# supybot.networks.oftc.nick: 
+
+# ###
+# # Determines what password will be used on oftc. Yes, we know that
+# # technically passwords are server-specific and not network-specific,
+# # but this is the best we can do right now.
+# #
+# # Default value: 
+# ###
+# supybot.networks.oftc.password: 
+
+# ###
+# # Deprecated config value, keep it to False.
+# #
+# # Default value: False
+# ###
+# supybot.networks.oftc.requireStarttls: False
+
+# ###
+# # Determines what SASL ECDSA key (if any) will be used on oftc. The
+# # public key must be registered with NickServ for SASL ECDSA-
+# # NIST256P-CHALLENGE to work.
+# #
+# # Default value: 
+# ###
+# supybot.networks.oftc.sasl.ecdsa_key: 
+
+# ###
+# # Determines what SASL mechanisms will be tried and in which order.
+# #
+# # Default value: ecdsa-nist256p-challenge external plain
+# ###
+# supybot.networks.oftc.sasl.mechanisms: ecdsa-nist256p-challenge external plain
+
+# ###
+# # Determines what SASL password will be used on oftc.
+# #
+# # Default value: 
+# ###
+# supybot.networks.oftc.sasl.password: 
+
+# ###
+# # Determines whether the bot will abort the connection if the none of
+# # the enabled SASL mechanism succeeded.
+# #
+# # Default value: False
+# ###
+# supybot.networks.oftc.sasl.required: False
+
+# ###
+# # Determines what SASL username will be used on oftc. This should be the
+# # bot's account name.
+# #
+# # Default value: 
+# ###
+# supybot.networks.oftc.sasl.username: 
+
+# ###
+# # Space-separated list of servers the bot will connect to for oftc. Each
+# # will be tried in order, wrapping back to the first when the cycle is
+# # completed.
+# #
+# # Default value: 
+# ###
+# supybot.networks.oftc.servers: irc.oftc.net:6697
+
+# ###
+# # If not empty, determines the hostname:port of the socks proxy that
+# # will be used to connect to this network.
+# #
+# # Default value: 
+# ###
+# supybot.networks.oftc.socksproxy: 
+
+# ###
+# # Determines whether the bot will attempt to connect with SSL sockets to
+# # oftc.
+# #
+# # Default value: True
+# ###
+# supybot.networks.oftc.ssl: True
+
+# ###
+# # A certificate that is trusted to verify certificates of this network
+# # (aka. Certificate Authority).
+# #
+# # Default value: 
+# ###
+# supybot.networks.oftc.ssl.authorityCertificate: 
+
+# ###
+# # Space-separated list of fingerprints of trusted certificates for this
+# # network. Supported hash algorithms are: md5, sha1, sha224, sha256,
+# # sha384, and sha512. If non-empty, Certification Authority signatures
+# # will not be used to verify certificates.
+# #
+# # Default value:  
+# ###
+# supybot.networks.oftc.ssl.serverFingerprints:  
+
+# ###
+# # Determines what user modes the bot will request from the server when
+# # it first connects. If empty, defaults to supybot.protocols.irc.umodes
+# #
+# # Default value: 
+# ###
+# supybot.networks.oftc.umodes: 
+
+# ###
+# # Determines the real name which the bot sends to the server. If empty,
+# # defaults to supybot.user
+# #
+# # Default value: 
+# ###
+# supybot.networks.oftc.user: 

playbooks/roles/limnoria/files/docker-compose.yaml

@@ -0,0 +1,18 @@
+# Version 2 is the latest that is supported by docker-compose in
+# Ubuntu Xenial.
+version: '2'
+
+services:
+  ircbot:
+    image: docker.io/opendevorg/ircbot:latest
+    network_mode: host
+    restart: always
+    logging:
+      driver: syslog
+      options:
+        tag: "docker-ircbot"
+    environment:
+      # This allows the meetbot plugin to find our config
+      PYTHONPATH: /var/lib/limnoria/ircmeeting
+    volumes:
+      - /var/lib/limnoria:/var/lib/limnoria

playbooks/roles/limnoria/handlers/main.yaml

@@ -0,0 +1,4 @@
+- name: restart apache2
+  service:
+    name: apache2
+    state: restarted

playbooks/roles/limnoria/tasks/main.yaml

@@ -0,0 +1,72 @@
+- name: Ensure /var/lib/limnoria directories
+  file:
+    state: directory
+    path: '/var/lib/{{ item }}'
+    mode: 0755
+  loop:
+    - limnoria
+    - limnoria/opendev
+
+- name: Put limnoira config in place
+  template:
+    src: limnoria.config.j2
+    dest: /var/lib/limnoria/limnoria.config
+    owner: root
+    group: root
+    mode: 0600
+
+- name: Ensure /var/lib/limnoria/ircmeeting directory
+  file:
+    state: directory
+    path: /var/lib/limnoria/ircmeeting
+    mode: 0755
+
+- name: Put meetbot local config in place
+  template:
+    src: meetingLocalConfig.py.j2
+    dest: /var/lib/limnoria/ircmeeting/meetingLocalConfig.py
+    owner: root
+    group: root
+    mode: 0600
+
+- name: Ensure /etc/ircbot-docker directory
+  file:
+    state: directory
+    path: /etc/ircbot-docker
+    mode: 0755
+
+- name: Setup webserver
+  include_tasks: webserver.yaml
+
+- name: Put docker-compose file in place
+  copy:
+    src: docker-compose.yaml
+    dest: /etc/ircbot-docker/docker-compose.yaml
+    owner: root
+    group: root
+    mode: 0644
+
+- name: 'Install logs2html cron job'
+  cron:
+    name: 'opendev {{ item.key }} logs2html'
+    state: present
+    user: root
+    job: >-
+        /usr/local/bin/docker-compose -f /etc/ircbot-docker/docker-compose.yaml exec -T ircbot
+        bash -c "find /var/lib/limnoria/opendev/logs/ChannelLogger/{{ item.key }}/ -mindepth 1 -maxdepth 1 -type d | xargs -n1 logs2html"
+  loop: '{{ limnoria_network_config | dict2items }}'
+  no_log: True
+
+- name: Run docker-compose pull
+  shell:
+    cmd: docker-compose pull
+    chdir: /etc/ircbot-docker/
+
+- name: Run docker-compose up
+  shell:
+    cmd: "docker-compose up -d"
+    chdir: /etc/ircbot-docker/
+
+- name: Run docker prune to cleanup unneeded images
+  shell:
+    cmd: docker image prune -f

playbooks/roles/limnoria/tasks/webserver.yaml

@@ -0,0 +1,33 @@
+- name: Install Apache
+  package:
+    name:
+      - apache2
+      - apache2-utils
+    state: present
+
+- name: Apache 2 ssl module
+  apache2_module:
+    state: present
+    name: ssl
+
+- name: Rewrite module
+  apache2_module:
+    state: present
+    name: rewrite
+
+- name: Create virtualhost
+  template:
+    src: vhost.conf.j2
+    dest: /etc/apache2/sites-available/001-eavesdrop.conf
+
+- name: Disable default site
+  command: a2dissite 000-default.conf
+  args:
+    removes: /etc/apache2/sites-enabled/000-default.conf
+
+- name: Enable mirror virtual host
+  command: a2ensite 001-eavesdrop
+  args:
+    creates: /etc/apache2/sites-enabled/001-eavesdrop.conf
+  notify:
+    - restart apache2

playbooks/roles/limnoria/templates/meetingLocalConfig.py.j2

@@ -0,0 +1,5 @@
+class Config(object):
+    # These two are **required**:
+    logFileDir = '/var/lib/limnoria/opendev/meetings/'
+    logUrlPrefix = 'https://eavesdrop.opendev.org/meetings/'
+    filenamePattern = '%(meetingname)s/%%Y/%(meetingname)s.%%F-%%H.%%M'

playbooks/roles/limnoria/templates/vhost.conf.j2

@@ -0,0 +1,56 @@
+<VirtualHost *:80>
+  ServerName {{ inventory_hostname }}
+
+  ErrorLog /var/log/apache2/{{ inventory_hostname }}_error.log
+  LogLevel warn
+  CustomLog /var/log/apache2/{{ inventory_hostname }}_access.log combined
+  ServerSignature Off
+
+  Redirect / https://{{ inventory_hostname }}/
+
+</VirtualHost>
+
+
+<VirtualHost *:443>
+  ServerName {{ inventory_hostname }}
+
+  SSLEngine on
+
+  SSLCertificateFile      /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.cer
+  SSLCertificateKeyFile   /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.key
+  SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer
+
+  SSLProtocol All -SSLv2 -SSLv3
+  # Note: this list should ensure ciphers that provide forward secrecy
+  SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
+  SSLHonorCipherOrder on
+
+  DocumentRoot /var/log/nodepool/builds
+  <Directory /var/log/nodepool/builds>
+    Options Indexes FollowSymLinks MultiViews
+    AllowOverride None
+    Require all granted
+  </Directory>
+
+  # Channel logs
+  Alias /irclogs /var/lib/limnoria/opendev/logs/ChannelLogger/oftc/
+  <Directory /var/lib/limnoria/opendev/logs/ChannelLogger/oftc/>
+    Options Indexes FollowSymLinks MultiViews
+    AllowOverride None
+    Require all granted
+  </Directory>
+
+  # Meeting logs
+  Alias /meetings /var/lib/limnoira/opendev/meetings/
+  <Directory  /var/lib/limnoira/opendev/meetings/>
+    Options Indexes FollowSymLinks MultiViews
+    AllowOverride None
+    Require all granted
+  </Directory>
+
+  ErrorLog /var/log/apache2/{{ inventory_hostname }}_error.log
+  LogLevel warn
+  CustomLog /var/log/apache2/{{ inventory_hostname }}_access.log combined
+  ServerSignature Off
+
+</VirtualHost>

playbooks/service-eavesdrop.yaml

@@ -7,3 +7,4 @@
     - sync-project-config
     - accessbot
     - gerritbot
+    - limnoria

zuul.d/docker-images/ircbot.yaml

@@ -3,6 +3,10 @@
     name: system-config-build-image-ircbot
     description: Build a ircbot image.
     parent: system-config-build-image
+    requires: &ircbot_requires
+      - python-base-3.9-container-image
+      - python-builder-3.9-container-image
+    provides: ircbot-container-image
     required-projects: &ircbot_required_projects
       - opendev/meetbot
     vars: &ircbot_vars
@@ -18,9 +22,12 @@
     name: system-config-upload-image-ircbot
     description: Build and upload a ircbot image.
     parent: system-config-upload-image
+    requires: *ircbot_requires
+    provides: ircbot-container-image
     required-projects: *ircbot_required_projects
     vars: *ircbot_vars
     files: *ircbot_files
+
 - job:
     name: system-config-promote-image-ircbot
     description: Promote a previously published ircbot image to latest.

zuul.d/project.yaml

@@ -20,6 +20,8 @@
               - name: opendev-buildset-registry
               - name: system-config-build-image-accessbot
                 soft: true
+              - name: system-config-build-image-ircbot
+                soft: true
         - system-config-run-codesearch:
             dependencies:
               - name: opendev-buildset-registry
@@ -147,6 +149,8 @@
               - name: opendev-buildset-registry
               - name: system-config-upload-image-accessbot
                 soft: true
+              - name: system-config-upload-image-ircbot
+                soft: true
         - system-config-run-codesearch:
             dependencies:
               - name: opendev-buildset-registry

zuul.d/system-config-run.yaml

@@ -130,6 +130,7 @@
     requires:
       - accessbot-container-image
       - gerritbot-container-image
+      - ircbot-container-image
     nodeset:
       nodes:
         - name: bridge.openstack.org
@@ -138,13 +139,22 @@
           label: ubuntu-focal
     vars:
       run_playbooks:
+        - playbooks/letsencrypt.yaml
         - playbooks/service-eavesdrop.yaml
+    host-vars:
+      eavesdrop01.opendev.org:
+        host_copy_output:
+          '/var/lib/limnoria': logs
+          '/var/log/apache2': logs
+          '/var/log/acme.sh': logs
+          '/etc/apache2': logs
     files:
       - playbooks/service-eavesdrop.yaml
       - playbooks/run-accessbot.yaml
       - inventory/service/group_vars/eavesdrop.yaml
       - playbooks/roles/install-docker
       - playbooks/roles/accessbot
+      - playbooks/roles/limnoria
       - playbooks/roles/logrotate
       - playbooks/zuul/templates/group_vars/eavesdrop.yaml.j2
       - docker/accessbot/