fix(rsyslog): fix maas-syslog failures in Noble containers

Two separate rsyslog problems prevent maas-syslog.service from starting
in MAAS region and rack controller containers on Noble hosts.

Problem 1 — imklog/chown capability restrictions:
rsyslogd fails because imklog requires CAP_SYSLOG to read /proc/kmsg
and $FileOwner/$FileGroup require CAP_CHOWN. Noble host kernels enforce
these restrictions more strictly than Jammy. Fixed by patching
/etc/rsyslog.conf at image build time to comment out these directives.
The same patch is applied in db-sync, which restarts rsyslog before
running region DB migrations.

Problem 2 — AppArmor blocks /var/lib/maas/rsyslog.conf:
On Noble the rsyslogd AppArmor profile was renamed from
/etc/apparmor.d/usr.sbin.rsyslogd to /etc/apparmor.d/rsyslogd.
maas-common.postinst only checks the old path, so it never reloads
the profile and the MAAS-specific rules in
/etc/apparmor.d/rsyslog.d/maas (which allow rsyslogd to read
/var/lib/maas/rsyslog.conf) are never loaded into the host kernel.
Fixed by two complementary measures:
- start.sh now loads the rsyslogd AppArmor profile via apparmor_parser
  before exec-ing systemd init. The container's SYS_ADMIN capability
  allows apparmor_parser to push the updated profile (including the
  MAAS rules) into the host kernel via the shared AppArmor securityfs.
- values.yaml sets AppArmor to unconfined for maas-region and maas-rack
  pods via pod.mandatory_access_control, providing a belt-and-suspenders
  guarantee that rsyslogd is never blocked regardless of the host profile
  state.

Also fixes a typo volueMounts -> volumeMounts in the maas-syslog sidecar
container definition in statefulset-region.yaml (the sidecar is disabled
by default but the typo would prevent volume mounts from applying if
manifests.maas_syslog were enabled).

Change-Id: Ieea22c3f75d9cd23844dc11e9a49f70a8ccb5a20
Signed-off-by: Sergiy Markin <smarkin@mirantis.com>

.zuul.yaml

@@ -25,7 +25,8 @@
         # - maas-airskiff-deployment-noble-kubeadm
         # - maas-airskiff-deployment-jammy-kubeadm
         # - maas-airskiff-deployment-focal-kubeadm
-        - maas-airskiff-deployment-noble-kubeadm-maas-upgrade
+        - maas-airskiff-deployment-noble-jammyhost-kubeadm-maas-upgrade
+        - maas-airskiff-deployment-noble-noblehost-kubeadm-maas-upgrade
     gate:
       jobs:
         - airship-maas-lint-ws
@@ -281,7 +282,7 @@
 #       zuul_treasuremap_relative_path: ../../airship/treasuremap
 
 - job:
-    name: maas-airskiff-deployment-noble-kubeadm-maas-upgrade
+    name: maas-airskiff-deployment-noble-noblehost-kubeadm-maas-upgrade
     description: |
       Deploy maas using Airskiff and submitted Shipyard changes.
     parent: treasuremap-airskiff-deploy-maas-drydock-base
@@ -304,6 +305,29 @@
       zuul_treasuremap_relative_path: ../../airship/treasuremap
 
 
+- job:
+    name: maas-airskiff-deployment-noble-jammyhost-kubeadm-maas-upgrade
+    description: |
+      Deploy maas using Airskiff and submitted Shipyard changes.
+    parent: treasuremap-airskiff-deploy-maas-drydock-base
+    nodeset: treasuremap-airskiff-5nodes-ubuntu_jammy
+    required-projects:
+      - name: airship/treasuremap
+        override-checkout: v1.9
+    roles:
+      - zuul: airship/treasuremap
+    vars:
+      treasuremap_ref: v1.9
+      ENABLE_SLEEP: false
+      ENABLE_MAAS_UPGRADE: true
+      CLONE_MAAS: false
+      MAKE_MAAS_IMAGES: true
+      helm_version: 4.2.0
+      HELM_ARTIFACT_URL: https://get.helm.sh/helm-v4.2.0-linux-amd64.tar.gz
+      DISTRO: ubuntu_noble
+      gate_scripts_relative_path: ../../airship/treasuremap
+      zuul_treasuremap_relative_path: ../../airship/treasuremap
+
 - secret:
     name: airship_maas_quay_creds
     data:

charts/maas/templates/bin/_db-sync.sh.tpl

@@ -18,6 +18,14 @@ set -ex
 
 # error: rsyslog: Unrecognized service
 rm -f /var/run/rsyslogd.pid || true
+# imklog requires CAP_SYSLOG which is not available in containers
+sed -i '/imklog/s/^/#/' /etc/rsyslog.conf
+# Disable file ownership directives that require CAP_CHOWN (not available in containers)
+sed -i '/\$FileOwner/s/^/#/' /etc/rsyslog.conf
+sed -i '/\$FileGroup/s/^/#/' /etc/rsyslog.conf
+# Pre-create log files so rsyslog (running as nobody) can write without chown
+touch /var/log/syslog /var/log/auth.log
+chmod 666 /var/log/syslog /var/log/auth.log
 service rsyslog restart || true
 
 maas-region dbupgrade

charts/maas/templates/bin/_start.sh.tpl

@@ -57,4 +57,18 @@ if [[ $sh_set = false ]]; then
   exit 1
 fi
 set -e
+# Fix AppArmor preventing rsyslogd from reading /var/lib/maas/rsyslog.conf.
+# On Noble the profile was renamed from usr.sbin.rsyslogd to rsyslogd, so
+# maas-common.postinst (which only checks the old path) never reloads it.
+# Reload the profile from the container filesystem (which has the MAAS rules
+# in rsyslog.d/maas) into the shared AppArmor securityfs before systemd starts.
+# --skip-read-cache forces recompile from source, bypassing a cached binary
+# profile that may predate the MAAS rules installation.
+if [ -d /sys/kernel/security/apparmor ] && command -v apparmor_parser >/dev/null 2>&1; then
+  for _p in /etc/apparmor.d/rsyslogd /etc/apparmor.d/usr.sbin.rsyslogd; do
+    [ -f "$_p" ] && { apparmor_parser --replace --skip-read-cache --write-cache "$_p" || true; break; }
+  done
+  unset _p
+fi
+
 exec /sbin/init --log-target=console 3>&1

charts/maas/templates/statefulset-region.yaml

@@ -122,7 +122,7 @@ spec:
               value: "/etc/rsyslog.conf"
             - name: LOGFILE
               value: {{ printf "%s/%s" .Values.conf.syslog.logpath .Values.conf.syslog.logfile | quote }}
-          volueMounts:
+          volumeMounts:
             - mountPath: /etc/rsyslog.conf
               name: maas-etc
               subPath: rsyslog.conf

charts/maas/values.yaml

@@ -438,6 +438,8 @@ pod:
       container:
         maas_rack:
           readOnlyRootFilesystem: false
+          appArmorProfile:
+            type: Unconfined
           capabilities:
             add:
               - "DAC_READ_SEARCH"
@@ -455,6 +457,8 @@ pod:
           readOnlyRootFilesystem: false
         maas_region:
           readOnlyRootFilesystem: false
+          appArmorProfile:
+            type: Unconfined
           capabilities:
             add:
               - "SYS_ADMIN"

images/maas-rack-controller-noble/Dockerfile

@@ -78,6 +78,13 @@ RUN cd /usr/lib/python3/dist-packages/provisioningserver/drivers/power && patch
 RUN cd /usr/lib/python3/dist-packages/provisioningserver/drivers/power && patch redfish.py < /tmp/3.6_redfish_retries.patch
 RUN cd /usr/lib/python3/dist-packages/provisioningserver/templates/dns && patch named.conf.options.inside.maas.template < /tmp/3.6_transfer_trusted_only.patch
 
+# Disable rsyslog modules that require privileges not available in containers.
+# imklog needs CAP_SYSLOG to read /proc/kmsg; $FileOwner/$FileGroup need CAP_CHOWN.
+# Noble host kernels enforce these restrictions more strictly than Jammy.
+RUN sed -i '/imklog/s/^/#/' /etc/rsyslog.conf \
+ && sed -i '/\$FileOwner/s/^/#/' /etc/rsyslog.conf \
+ && sed -i '/\$FileGroup/s/^/#/' /etc/rsyslog.conf
+
 # echo journalctl logs to the container's stdout
 COPY scripts/journalctl-to-tty.service /etc/systemd/system/journalctl-to-tty.service
 RUN systemctl enable journalctl-to-tty.service
@@ -93,6 +100,8 @@ RUN grep -i "^pidfile " /etc/chrony/chrony.conf && \
     sed -i 's|^pidfile .*|pidfile /run/chrony/chronyd.pid|' /etc/chrony/chrony.conf || \
     echo 'pidfile /run/chrony/chronyd.pid' >> /etc/chrony/chrony.conf
 COPY scripts/override.chrony.conf /etc/systemd/system/chrony.service.d/override.conf
+RUN mkdir -p /etc/systemd/system/maas-syslog.service.d
+COPY scripts/override.maas-syslog.conf /etc/systemd/system/maas-syslog.service.d/override.conf
 
 # fix httpproxy.sock permission
 COPY scripts/maas-http-watcher.service /etc/systemd/system/maas-http-watcher.service

images/maas-rack-controller-noble/scripts/override.maas-syslog.conf

@@ -0,0 +1,2 @@
+[Service]
+AppArmorProfile=unconfined

images/maas-region-controller-noble/Dockerfile

@@ -83,6 +83,13 @@ RUN cd /usr/lib/python3/dist-packages/maasserver/models && patch ownerdata.py <
 RUN cd /usr/lib/python3/dist-packages/provisioningserver/templates/dns && patch named.conf.options.inside.maas.template < /tmp/3.6_transfer_trusted_only.patch
 RUN cd /usr/lib/python3/dist-packages/maasserver/workflow && patch __init__.py < /tmp/3.6_temporal_workflow_policy.patch
 
+# Disable rsyslog modules that require privileges not available in containers.
+# imklog needs CAP_SYSLOG to read /proc/kmsg; $FileOwner/$FileGroup need CAP_CHOWN.
+# Noble host kernels enforce these restrictions more strictly than Jammy.
+RUN sed -i '/imklog/s/^/#/' /etc/rsyslog.conf \
+ && sed -i '/\$FileOwner/s/^/#/' /etc/rsyslog.conf \
+ && sed -i '/\$FileGroup/s/^/#/' /etc/rsyslog.conf
+
 # echo journalctl logs to the container's stdout
 COPY journalctl-to-tty.service /etc/systemd/system/journalctl-to-tty.service
 RUN systemctl enable journalctl-to-tty.service
@@ -98,6 +105,8 @@ RUN grep -i "^pidfile " /etc/chrony/chrony.conf && \
     sed -i 's|^pidfile .*|pidfile /run/chrony/chronyd.pid|' /etc/chrony/chrony.conf || \
     echo 'pidfile /run/chrony/chronyd.pid' >> /etc/chrony/chrony.conf
 COPY override.chrony.conf /etc/systemd/system/chrony.service.d/override.conf
+RUN mkdir -p /etc/systemd/system/maas-syslog.service.d
+COPY override.maas-syslog.conf /etc/systemd/system/maas-syslog.service.d/override.conf
 
 # initalize systemd
 CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=console 3>&1"]

images/maas-region-controller-noble/override.maas-syslog.conf

@@ -0,0 +1,2 @@
+[Service]
+AppArmorProfile=unconfined