Decouple pci-irq-affinity-agent service

This change is part of an initiative to decouple the pci-irq-affinity
agent process from the platform by converting it into a resource to
be deployed along with stx-openstack application.

Depends-on: https://review.opendev.org/c/starlingx/utilities/+/814031
Story: 2009299
Task: 43656

Change-Id: Iefc1106e01cbfc874119e16b610e48a629771db1
Signed-off-by: Heitor Matsui <HeitorVieira.Matsui@windriver.com>
This commit is contained in:
Heitor Matsui 2021-10-08 16:31:17 -03:00
parent 9a86882cda
commit 7f9029b21f
16 changed files with 555 additions and 2 deletions

View File

@ -1,6 +1,6 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (c) 2019-2020 Wind River Systems, Inc.
# Copyright (c) 2019-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -33,6 +33,7 @@ from k8sapp_openstack.helm.neutron import NeutronHelm
from k8sapp_openstack.helm.nginx_ports_control import NginxPortsControlHelm
from k8sapp_openstack.helm.nova import NovaHelm
from k8sapp_openstack.helm.nova_api_proxy import NovaApiProxyHelm
from k8sapp_openstack.helm.pci_irq_affinity_agent import PciIrqAffinityAgentHelm
from k8sapp_openstack.helm.openvswitch import OpenvswitchHelm
from k8sapp_openstack.helm.panko import PankoHelm
from k8sapp_openstack.helm.placement import PlacementHelm
@ -94,6 +95,7 @@ class OpenstackArmadaManifestOperator(base.ArmadaManifestOperator):
NginxPortsControlHelm.CHART: CHART_GROUP_INGRESS_OS,
NovaHelm.CHART: CHART_GROUP_COMPUTE_KIT,
NovaApiProxyHelm.CHART: CHART_GROUP_COMPUTE_KIT,
PciIrqAffinityAgentHelm.CHART: CHART_GROUP_COMPUTE_KIT,
OpenvswitchHelm.CHART: CHART_GROUP_COMPUTE_KIT,
PankoHelm.CHART: CHART_GROUP_TELEMETRY,
PlacementHelm.CHART: CHART_GROUP_COMPUTE_KIT,
@ -126,6 +128,7 @@ class OpenstackArmadaManifestOperator(base.ArmadaManifestOperator):
NginxPortsControlHelm.CHART: 'openstack-nginx-ports-control',
NovaHelm.CHART: 'openstack-nova',
NovaApiProxyHelm.CHART: 'openstack-nova-api-proxy',
PciIrqAffinityAgentHelm.CHART: 'openstack-pci-irq-affinity-agent',
OpenvswitchHelm.CHART: 'openstack-openvswitch',
PankoHelm.CHART: 'openstack-panko',
PSPRolebindingHelm.CHART: 'openstack-psp-rolebinding',

View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2019-2020 Wind River Systems, Inc.
# Copyright (c) 2019-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -29,6 +29,7 @@ HELM_CHART_NEUTRON = 'neutron'
HELM_CHART_NGINX_PORTS_CONTROL = "nginx-ports-control"
HELM_CHART_NOVA = 'nova'
HELM_CHART_NOVA_API_PROXY = 'nova-api-proxy'
HELM_CHART_PCI_IRQ_AFFINITY_AGENT = 'pci-irq-affinity-agent'
HELM_CHART_OPENVSWITCH = 'openvswitch'
HELM_CHART_PANKO = 'panko'
HELM_CHART_PLACEMENT = 'placement'

View File

@ -0,0 +1,55 @@
#
# Copyright (c) 2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
from oslo_log import log as logging
from k8sapp_openstack.common import constants as app_constants
from k8sapp_openstack.helm import openstack
from sysinv.common import exception
from sysinv.helm import common
LOG = logging.getLogger(__name__)
class PciIrqAffinityAgentHelm(openstack.OpenstackBaseHelm):
"""Class to encapsulate helm operations for the PCI IRQ affinity agent chart"""
CHART = app_constants.HELM_CHART_PCI_IRQ_AFFINITY_AGENT
def __init__(self, operator):
super(PciIrqAffinityAgentHelm, self).__init__(operator)
def get_overrides(self, namespace=None):
overrides = {
common.HELM_NS_OPENSTACK: {
'conf': {
'endpoints': self._get_endpoints_overrides()
}
}
}
if namespace in self.SUPPORTED_NAMESPACES:
return overrides[namespace]
elif namespace:
raise exception.InvalidHelmNamespace(chart=self.CHART,
namespace=namespace)
else:
return overrides
def _get_endpoints_overrides(self):
nova_oslo_messaging_data = self._get_endpoints_oslo_messaging_overrides(
'nova',
['nova']
)['nova']
overrides = {
'rabbit': {
'rabbit_userid': nova_oslo_messaging_data['username'],
'rabbit_password': nova_oslo_messaging_data['password'],
},
}
return overrides

View File

@ -62,6 +62,7 @@ systemconfig.helm_plugins.stx_openstack =
027_fm-rest-api = k8sapp_openstack.helm.fm_rest_api:FmRestApiHelm
028_dcdbsync = k8sapp_openstack.helm.dcdbsync:DcdbsyncHelm
029_openstack-psp-rolebinding = k8sapp_openstack.helm.psp_rolebinding:PSPRolebindingHelm
030_pci-irq-affinity-agent = k8sapp_openstack.helm.pci_irq_affinity_agent:PciIrqAffinityAgentHelm
systemconfig.armada.manifest_ops =
stx-openstack = k8sapp_openstack.armada.manifest_openstack:OpenstackArmadaManifestOperator

View File

@ -42,6 +42,7 @@ helm repo add local http://localhost:8879/charts
# Make the charts. These produce a tgz file
cd helm-charts
make nova-api-proxy
make pci-irq-affinity-agent
make garbd
make keystone-api-proxy
make fm-rest-api

View File

@ -0,0 +1,4 @@
apiVersion: v1
description: PCI IRQ Affinity Agent
name: pci-irq-affinity-agent
version: 0.1.0

View File

@ -0,0 +1,10 @@
#
# Copyright (c) 2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
dependencies:
- name: helm-toolkit
repository: http://localhost:8879/charts
version: ">= 0.1.0"

View File

@ -0,0 +1,85 @@
#!/usr/bin/env python
#
# Copyright (c) 2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
"""
Health probe script PCI IRQ Affinity Agent.
Script returns failure to Kubernetes only when
a. Fails to call nova to get instances
sys.stderr.write() writes to pod's events on failures.
Usage example:
# python health-probe.py
"""
import json
import os
import signal
import sys
import psutil
from pci_irq_affinity.nova_provider import novaClient
def test_rabbit_connection():
rabbit_ok = False
for p in psutil.process_iter():
if 'python' in ' '.join(p.cmdline()):
conns = p.connections()
for c in conns:
if c.raddr[1] == 5672 and c.status == 'ESTABLISHED':
rabbit_ok = True
return rabbit_ok
def test_nova_availability():
try:
novaClient.get_nova()
novaClient.get_instances({})
except:
return False
return True
def check_pid_running(pid):
if psutil.pid_exists(int(pid)):
return True
else:
return False
if __name__ == "__main__":
if "liveness-probe" in ','.join(sys.argv):
pidfile = "/tmp/liveness.pid" # nosec
else:
pidfile = "/tmp/readiness.pid" # nosec
data = {}
if os.path.isfile(pidfile):
with open(pidfile, 'r') as f:
data = json.load(f)
if check_pid_running(data['pid']):
if data['exit_count'] > 1:
# Third time in, kill the previous process
os.kill(int(data['pid']), signal.SIGTERM)
sys.exit(1)
else:
data['exit_count'] = data['exit_count'] + 1
with open(pidfile, 'w') as f:
json.dump(data, f)
sys.exit(0)
data['pid'] = os.getpid()
data['exit_count'] = 0
with open(pidfile, 'w') as f:
json.dump(data, f)
if test_rabbit_connection() and test_nova_availability():
sys.exit(0) # return success
else:
sys.exit(1)

View File

@ -0,0 +1,13 @@
#!/bin/bash
#
# Copyright (c) 2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# Script to patch the keyring library so that it allows automation
# by not asking for user input on the keyring password
# This 'change' was taken from the platform keyring library
KEYRING_LIB=$(find / -name file.py)
sed -i '/self.keyring_key *= *getpass.getpass(/,/)/s/^/#/;/self.keyring_key *= *getpass.getpass/i\ # TAKEN FROM PLATFORM KEYRING CODE\n\ self.keyring_key = "Please set a password for your new keyring: "' $KEYRING_LIB

View File

@ -0,0 +1,20 @@
#!/usr/bin/env python
#
# Copyright (c) 2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
"""
Startup script for PCI IRQ Affinity Agent.
Usage example:
# python start.py
"""
from pci_irq_affinity import agent
if __name__ == '__main__':
agent.process_main()

View File

@ -0,0 +1,11 @@
#!/bin/bash
#
# Copyright (c) 2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# Script to encapsulate the starting routines
sh -c /tmp/patch_keyring.sh
python /tmp/start.py

View File

@ -0,0 +1,26 @@
{{/*
#
# Copyright (c) 2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
*/}}
{{- if .Values.manifests.configmap_bin }}
{{- $configMapName := "pci-irq-affinity-agent-bin" }}
{{- $envAll := . }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ $configMapName }}
data:
start.py: |
{{ tuple "bin/_start.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
health-probe.py: |
{{ tuple "bin/_health-probe.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
patch_keyring.sh: |
{{ tuple "bin/_patch_keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
start.sh: |
{{ tuple "bin/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}

View File

@ -0,0 +1,21 @@
{{/*
#
# Copyright (c) 2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
*/}}
{{- if .Values.manifests.configmap_etc }}
{{- $configMapName := "pci-irq-affinity-agent-etc" }}
{{- $envAll := . }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ $configMapName }}
type: Opaque
data:
{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.template "key" "config.ini" "format" "Secret") | indent 2 }}
{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.keyring "key" "keyringrc.cfg" "format" "Secret") | indent 2 }}
{{- end }}

View File

@ -0,0 +1,140 @@
{{/*
#
# Copyright (c) 2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
*/}}
{{- if .Values.manifests.daemonset }}
{{- $daemonset := "pci-irq-affinity-agent" }}
{{- $configMapName := "pci-irq-affinity-agent-etc" }}
{{- $binConfigMapName := "pci-irq-affinity-agent-bin" }}
{{- $envAll := . }}
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: {{ $daemonset }}
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
labels:
{{ tuple $envAll .Chart.Name $daemonset | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
spec:
selector:
matchLabels:
{{ tuple $envAll .Chart.Name $daemonset | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
template:
metadata:
labels:
{{ tuple $envAll .Chart.Name $daemonset | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
spec:
nodeSelector:
{{ .Values.labels.agent.pci_irq_affinity_agent.node_selector_key }}: {{ .Values.labels.agent.pci_irq_affinity_agent.node_selector_value }}
tolerations:
{{ toYaml .Values.tolerations | indent 8 }}
containers:
- name: pci-irq-affinity-agent
image: {{ .Values.images.tags.pci_irq_affinity_agent }}
imagePullPolicy: {{ .Values.images.pull_policy }}
command:
- /tmp/start.sh
{{- if .Values.pod.probes.readiness.pci_irq_affinity_agent.enabled }}
env:
- name: COMPUTE_HOSTNAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: COMPUTE_PCI_DEVICES
value: {{ .Values.conf.host.mount_path.pci_devices }}
- name: COMPUTE_IRQ
value: {{ .Values.conf.host.mount_path.irq }}
readinessProbe:
exec:
command:
- python
- /tmp/health-probe.py
initialDelaySeconds: 30
periodSeconds: 180
timeoutSeconds: 165
{{- end }}
{{- if .Values.pod.probes.liveness.pci_irq_affinity_agent.enabled }}
livenessProbe:
exec:
command:
- python
- /tmp/health-probe.py
- --liveness-probe
initialDelaySeconds: 60
periodSeconds: 60
timeoutSeconds: 55
{{- end }}
volumeMounts:
- name: {{ $configMapName }}
mountPath: /etc/pci_irq_affinity/config.ini
subPath: config.ini
readOnly: true
- name: {{ $configMapName }}
mountPath: /root/.local/share/python_keyring/keyringrc.cfg
subPath: keyringrc.cfg
readOnly: true
- name: keyring
mountPath: /root/.local/share/python_keyring/crypted_pass.cfg
readOnly: true
- name: libvirt-sock-ro
mountPath: /var/run/libvirt/libvirt-sock-ro
readOnly: true
- name: compute-pci-devices
mountPath: {{ .Values.conf.host.mount_path.pci_devices }}
readOnly: false
- name: compute-irq
mountPath: {{ .Values.conf.host.mount_path.irq }}
readOnly: false
- name: {{ $binConfigMapName }}
mountPath: /tmp/start.py
subPath: start.py
readOnly: true
- name: {{ $binConfigMapName }}
mountPath: /tmp/patch_keyring.sh
subPath: patch_keyring.sh
readOnly: true
- name: {{ $binConfigMapName }}
mountPath: /tmp/start.sh
subPath: start.sh
readOnly: true
- name: {{ $binConfigMapName }}
mountPath: /tmp/health-probe.py
subPath: health-probe.py
readOnly: true
{{- if .Values.manifests.certificates }}
{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.pci_irq_affinity_agent.agent.public | include "helm-toolkit.snippets.tls_volume_mount" | indent 10 }}
{{- end }}
volumes:
- name: {{ $configMapName }}
secret:
defaultMode: 0644
secretName: {{ $configMapName }}
- name: keyring
hostPath:
path: /opt/platform/.keyring/21.12/python_keyring/crypted_pass.cfg # TODO (hmatsui): dynamically get version (e.g. 21.12)
type: File
- name: libvirt-sock-ro
hostPath:
path: /var/run/libvirt/libvirt-sock-ro
type: Socket
- name: compute-pci-devices
hostPath:
path: /sys/bus/pci/devices
type: Directory
- name: compute-irq
hostPath:
path: /proc/irq
type: Directory
- name: {{ $binConfigMapName }}
configMap:
defaultMode: 0555
name: {{ $binConfigMapName }}
{{- if .Values.manifests.certificates }}
{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.pci_irq_affinity_agent.agent.public | include "helm-toolkit.snippets.tls_volume" | indent 6 }}
{{- end }}
{{- end }}

View File

@ -0,0 +1,122 @@
#
# Copyright (c) 2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# Default values for pci-irq-affinity-agent.
# This is a YAML-formatted file.
# Declare name/value pairs to be passed into your templates.
# name: value
release_group: null
images:
pull_policy: "IfNotPresent"
tags:
pci_irq_affinity_agent: docker.io/starlingx/stx-pci-irq-affinity-agent:master-centos-stable-latest
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
local_registry:
active: false
exclude:
- dep_check
labels:
agent:
pci_irq_affinity_agent:
node_selector_key: openstack-compute-node
node_selector_value: enabled
dependencies:
static:
pci_irq_affinity_agent:
services:
- endpoint: internal
service: keystone
- endpoint: internal
service: rabbitmq
- endpoint: internal
service: nova
pod:
security_context:
pci_irq_affinity_agent:
pod:
runAsUser: 0
container:
readOnlyRootFilesystem: true
probes:
readiness:
pci_irq_affinity_agent:
enabled: true
liveness:
pci_irq_affinity_agent:
enabled: true
conf:
endpoints:
identity:
name: keystone
auth:
admin:
username: admin
project_name: admin
user_domain_name: default
project_domain_name: default
cacert: /etc/ssl/certs/openstack-helm.crt
keystone:
openstack_enabled: True
openstack_auth_protocol: http
openstack_auth_host: keystone.openstack.svc.cluster.local
openstack_auth_port: 80
openstack_keyring_service: CGCS
rabbit:
rabbit_host: rabbitmq.openstack.svc.cluster.local
rabbit_port: 5672
rabbit_userid: nova
rabbit_password: password
rabbit_virtual_host: nova
template: |
[openstack]
openstack_enabled={{ .Values.conf.endpoints.keystone.openstack_enabled }}
username={{ .Values.conf.endpoints.identity.auth.admin.username }}
tenant={{ .Values.conf.endpoints.identity.auth.admin.username }}
authorization_protocol={{ .Values.conf.endpoints.keystone.openstack_auth_protocol }}
authorization_ip={{ .Values.conf.endpoints.keystone.openstack_auth_host }}
authorization_port={{ .Values.conf.endpoints.keystone.openstack_auth_port }}
user_domain_name={{ .Values.conf.endpoints.identity.auth.admin.user_domain_name }}
project_domain_name={{ .Values.conf.endpoints.identity.auth.admin.project_domain_name}}
project_name={{ .Values.conf.endpoints.identity.auth.admin.project_name }}
keyring_service={{ .Values.conf.endpoints.keystone.openstack_keyring_service }}
auth_url={{ .Values.conf.endpoints.keystone.openstack_auth_protocol }}://{{ .Values.conf.endpoints.keystone.openstack_auth_host }}/v3
{{- if .Values.manifests.certificates }}
cacert={{ .Values.conf.endpoints.identity.auth.admin.cacert }}
{{- end }}
[amqp]
host={{ .Values.conf.endpoints.rabbit.rabbit_host }}
port={{ .Values.conf.endpoints.rabbit.rabbit_port }}
user_id={{ .Values.conf.endpoints.rabbit.rabbit_userid }}
password={{ .Values.conf.endpoints.rabbit.rabbit_password }}
virt_host={{ .Values.conf.endpoints.rabbit.rabbit_virtual_host }}
keyring: |
[backend]
default-keyring=keyrings.alt.file.EncryptedKeyring
host:
mount_path:
pci_devices: /compute/sys/bus/pci/devices
irq: /compute/proc/irq
secrets:
tls:
pci_irq_affinity_agent:
agent:
public: nova-tls-public
tolerations: []
manifests:
certificates: false
configmap_bin: true
configmap_etc: true
daemonset: true

View File

@ -1393,6 +1393,45 @@ data:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: openstack-pci-irq-affinity-agent
data:
chart_name: pci-irq-affinity-agent
release: openstack-pci-irq-affinity-agent
namespace: openstack
wait:
timeout: 1800
labels:
release_group: osh-openstack-pci-irq-affinity-agent
test:
enabled: false
install:
no_hooks: false
upgrade:
no_hooks: false
pre:
delete:
- type: job
labels:
release_group: osh-openstack-pci-irq-affinity-agent
values:
images:
tags:
pci_irq_affinity_agent: docker.io/starlingx/stx-pci-irq-affinity-agent:master-centos-stable-latest
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
source:
type: tar
location: http://172.17.0.1/helm_charts/starlingx/pci-irq-affinity-agent-0.1.0.tgz
subpath: pci-irq-affinity-agent
reference: master
dependencies:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: openstack-neutron
@ -3924,6 +3963,7 @@ data:
- openstack-libvirt
- openstack-nova
- openstack-nova-api-proxy
- openstack-pci-irq-affinity-agent
- openstack-neutron
- openstack-placement
---