Implement neutron

- Stop installing Neutron on the physical host
- Migrate Nova RabbitMQ and Memcache to Kubernetes
- Added Neutron metadata agent to Kubernetes
- Stop running metering agent
- Moved DHCP agent to Kubernetes
- Migrate Neutron L3 agent to Kubernetes

Change-Id: I9dadb88b072183b6bdacb030e3585d1d17e9a479

chart/test-values.yaml

@@ -8,6 +8,7 @@ configMap:
   glance: {}
   placement: {}
   neutron: {}
+  nova: {}
   horizon:
     hostAliases:
     - hostnames:

config/samples/operator-config.yaml

@@ -17,6 +17,7 @@ data:
     chronyd: {}
     placement: {}
     neutron: {}
+    nova: {}
     backup:
       secretName: aws-backup
       url: s3://backups/

devstack/lib/barbican

@@ -163,7 +163,7 @@ function configure_barbican {
     iniset $BARBICAN_PASTE_CONF 'pipeline:barbican_api' pipeline 'barbican-api-keystone'
 
     # Set the keystone parameters
-    configure_auth_token_middleware $BARBICAN_CONF barbican $BARBICAN_AUTH_CACHE_DIR
+    configure_keystone_authtoken_middleware $BARBICAN_CONF barbican
 
     # NOTE(Alex): Operator stuff for memcached
     iniset $BARBICAN_CONF keystone_authtoken memcached_servers "mcrouter-memcached-barbican:11211"
@@ -180,11 +180,6 @@ function init_barbican {
     --from-file=/etc/barbican/barbican-api-paste.ini \
     --from-file=/etc/barbican/barbican-functional.conf
 
-    # Create cache dir
-    sudo mkdir -p $BARBICAN_AUTH_CACHE_DIR
-    sudo chown $STACK_USER $BARBICAN_AUTH_CACHE_DIR
-    rm -f $BARBICAN_AUTH_CACHE_DIR/*
-
     recreate_database barbican utf8
 }
 

devstack/lib/common

@@ -46,7 +46,6 @@ function kubernetes_rollout_restart {
 
 function kubernetes_ensure_resource {
 	local resource="$1"
-	kubectl logs deployment/openstack-operator -n default
 	for i in {1..120}; do
 		kubectl get $resource && break || sleep 5;
 	done

devstack/lib/neutron-legacy

@@ -14,6 +14,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+NEUTRON_STATE_PATH=/var/lib/neutron
 function configure_mutnauq {
 	_configure_neutron_common
 
@@ -61,6 +62,11 @@ function create_mutnauq_accounts {
 	create_service_user "neutron"
 }
 
+function install_mutnauq {
+	echo noop
+}
+export -f install_mutnauq
+
 function init_mutnauq {
 	echo noop
 }
@@ -71,6 +77,9 @@ function start_neutron_service_and_check {
 
 	kubectl -n openstack create secret generic neutron-config \
 		--from-file=/etc/neutron/neutron.conf \
+		--from-file=/etc/neutron/l3_agent.ini \
+		--from-file=/etc/neutron/dhcp_agent.ini \
+		--from-file=/etc/neutron/metadata_agent.ini \
 		--from-file=/etc/neutron/api-paste.ini \
 		--from-file=/etc/neutron/policy.json
 	kubectl -n openstack create secret generic neutron-ml2-config \
@@ -107,6 +116,18 @@ function start_mutnauq_l2_agent {
 }
 export -f start_neutron_agents
 
+function start_mutnauq_other_agents {
+	kubernetes_rollout_restart daemonset/neutron-dhcp-agent
+	kubernetes_rollout_status daemonset/neutron-dhcp-agent
+
+	kubernetes_rollout_restart daemonset/neutron-l3-agent
+	kubernetes_rollout_status daemonset/neutron-l3-agent
+
+	kubernetes_rollout_restart daemonset/neutron-metadata-agent
+	kubernetes_rollout_status daemonset/neutron-metadata-agent
+}
+export -f start_mutnauq_other_agents
+
 function _configure_neutron_common {
 	_create_neutron_conf_dir
 
@@ -151,11 +172,10 @@ function _configure_neutron_common {
 	NEUTRON_DATABASE_NAME=$(get_data_from_secret neutron-mysql openstack DATABASE)
 	iniset $NEUTRON_CONF database connection "mysql+pymysql://$NEUTRON_DATABASE_USER:$NEUTRON_DATABASE_PASSWORD@neutron-mysql-master/$NEUTRON_DATABASE_NAME?charset=utf8"
 
-	iniset $NEUTRON_CONF DEFAULT state_path $DATA_DIR/neutron
+	iniset $NEUTRON_CONF DEFAULT state_path $NEUTRON_STATE_PATH
 	iniset $NEUTRON_CONF DEFAULT use_syslog $SYSLOG
 	iniset $NEUTRON_CONF DEFAULT bind_host $Q_LISTEN_ADDRESS
-	iniset $NEUTRON_CONF oslo_concurrency lock_path $DATA_DIR/neutron/lock
+	iniset $NEUTRON_CONF oslo_concurrency lock_path $NEUTRON_STATE_PATH/lock
-
 	# NOTE(freerunner): Need to adjust Region Name for nova in multiregion installation
 	iniset $NEUTRON_CONF nova region_name $REGION_NAME
 
@@ -216,3 +236,18 @@ function _configure_neutron_service {
 	neutron_plugin_configure_service
 }
 export -f _configure_neutron_service
+
+function _neutron_ovs_base_add_bridge {
+    echo noop
+}
+export -f _neutron_ovs_base_add_bridge
+
+function _neutron_ovs_base_setup_bridge {
+    echo noop
+}
+export -f _neutron_ovs_base_setup_bridge
+
+function _neutron_ovs_base_configure_l3_agent {
+	echo noop
+}
+export -f _neutron_ovs_base_configure_l3_agent

devstack/lib/nova

@@ -0,0 +1,196 @@
+#!/bin/bash
+#
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#	 http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+function create_nova_conf {
+	# Remove legacy ``nova.conf``
+	rm -f $NOVA_DIR/bin/nova.conf
+
+	# (Re)create ``nova.conf``
+	rm -f $NOVA_CONF
+	iniset $NOVA_CONF DEFAULT debug "$ENABLE_DEBUG_LOG_LEVEL"
+	if [ "$NOVA_ALLOW_MOVE_TO_SAME_HOST" == "True" ]; then
+		iniset $NOVA_CONF DEFAULT allow_resize_to_same_host "True"
+	fi
+	iniset $NOVA_CONF wsgi api_paste_config "$NOVA_API_PASTE_INI"
+	iniset $NOVA_CONF DEFAULT rootwrap_config "$NOVA_CONF_DIR/rootwrap.conf"
+	iniset $NOVA_CONF filter_scheduler enabled_filters "$NOVA_FILTERS"
+	iniset $NOVA_CONF scheduler workers "$API_WORKERS"
+	iniset $NOVA_CONF neutron default_floating_pool "$PUBLIC_NETWORK_NAME"
+	if [[ $SERVICE_IP_VERSION == 6 ]]; then
+		iniset $NOVA_CONF DEFAULT my_ip "$HOST_IPV6"
+	else
+		iniset $NOVA_CONF DEFAULT my_ip "$HOST_IP"
+	fi
+	iniset $NOVA_CONF DEFAULT instance_name_template "${INSTANCE_NAME_PREFIX}%08x"
+	iniset $NOVA_CONF DEFAULT osapi_compute_listen "$NOVA_SERVICE_LISTEN_ADDRESS"
+	iniset $NOVA_CONF DEFAULT metadata_listen "$NOVA_SERVICE_LISTEN_ADDRESS"
+	iniset $NOVA_CONF DEFAULT shutdown_timeout $NOVA_SHUTDOWN_TIMEOUT
+
+	iniset $NOVA_CONF key_manager backend nova.keymgr.conf_key_mgr.ConfKeyManager
+
+	if is_fedora || is_suse; then
+		# nova defaults to /usr/local/bin, but fedora and suse pip like to
+		# install things in /usr/bin
+		iniset $NOVA_CONF DEFAULT bindir "/usr/bin"
+	fi
+
+	# only setup database connections and cache backend if there are services
+	# that require them running on the host. The ensures that n-cpu doesn't
+	# leak a need to use the db in a multinode scenario.
+	if is_service_enabled n-api n-cond n-sched; then
+		# If we're in multi-tier cells mode, we want our control services pointing
+		# at cell0 instead of cell1 to ensure isolation. If not, we point everything
+		# at the main database like normal.
+		if [[ "$CELLSV2_SETUP" == "singleconductor" ]]; then
+			local db="nova_cell1"
+		else
+			local db="nova_cell0"
+			# When in superconductor mode, nova-compute can't send instance
+			# info updates to the scheduler, so just disable it.
+			iniset $NOVA_CONF filter_scheduler track_instance_changes False
+		fi
+
+		iniset $NOVA_CONF database connection `database_connection_url $db`
+		iniset $NOVA_CONF api_database connection `database_connection_url nova_api`
+
+		# Cache related settings
+		# Those settings aren't really needed in n-cpu thus it is configured
+		# only on nodes which runs controller services
+		iniset $NOVA_CONF cache enabled $NOVA_ENABLE_CACHE
+		iniset $NOVA_CONF cache backend $CACHE_BACKEND
+		iniset $NOVA_CONF cache memcache_servers mcrouter-memcached-nova.openstack.svc.cluster.local
+	fi
+
+	if is_service_enabled n-api; then
+		if is_service_enabled n-api-meta; then
+			# If running n-api-meta as a separate service
+			NOVA_ENABLED_APIS=$(echo $NOVA_ENABLED_APIS | sed "s/,metadata//")
+		fi
+		iniset $NOVA_CONF DEFAULT enabled_apis "$NOVA_ENABLED_APIS"
+		if is_service_enabled tls-proxy && [ "$NOVA_USE_MOD_WSGI" == "False" ]; then
+			# Set the service port for a proxy to take the original
+			iniset $NOVA_CONF DEFAULT osapi_compute_listen_port "$NOVA_SERVICE_PORT_INT"
+			iniset $NOVA_CONF DEFAULT osapi_compute_link_prefix $NOVA_SERVICE_PROTOCOL://$NOVA_SERVICE_HOST:$NOVA_SERVICE_PORT
+		fi
+
+		configure_keystone_authtoken_middleware $NOVA_CONF nova
+	fi
+
+	if is_service_enabled cinder; then
+		configure_cinder_access
+	fi
+
+	if [ -n "$NOVA_STATE_PATH" ]; then
+		iniset $NOVA_CONF DEFAULT state_path "$NOVA_STATE_PATH"
+		iniset $NOVA_CONF oslo_concurrency lock_path "$NOVA_STATE_PATH"
+	fi
+	if [ -n "$NOVA_INSTANCES_PATH" ]; then
+		iniset $NOVA_CONF DEFAULT instances_path "$NOVA_INSTANCES_PATH"
+	fi
+	if [ "$SYSLOG" != "False" ]; then
+		iniset $NOVA_CONF DEFAULT use_syslog "True"
+	fi
+	if [ "$FORCE_CONFIG_DRIVE" != "False" ]; then
+		iniset $NOVA_CONF DEFAULT force_config_drive "$FORCE_CONFIG_DRIVE"
+	fi
+
+	# nova defaults to genisoimage but only mkisofs is available for 15.0+
+	if is_suse; then
+		iniset $NOVA_CONF DEFAULT mkisofs_cmd /usr/bin/mkisofs
+	fi
+
+	# Format logging
+	setup_logging $NOVA_CONF
+
+	iniset $NOVA_CONF upgrade_levels compute "auto"
+
+	write_uwsgi_config "$NOVA_UWSGI_CONF" "$NOVA_UWSGI" "/compute"
+	write_uwsgi_config "$NOVA_METADATA_UWSGI_CONF" "$NOVA_METADATA_UWSGI" "" "$SERVICE_LISTEN_ADDRESS:${METADATA_SERVICE_PORT}"
+
+	if is_service_enabled ceilometer; then
+		iniset $NOVA_CONF DEFAULT instance_usage_audit "True"
+		iniset $NOVA_CONF DEFAULT instance_usage_audit_period "hour"
+		iniset $NOVA_CONF DEFAULT notify_on_state_change "vm_and_task_state"
+	fi
+
+	# Set the oslo messaging driver to the typical default. This does not
+	# enable notifications, but it will allow them to function when enabled.
+	iniset $NOVA_CONF oslo_messaging_notifications driver "messagingv2"
+	iniset $NOVA_CONF oslo_messaging_notifications transport_url $(get_notification_url)
+	iniset $NOVA_CONF notifications notification_format "$NOVA_NOTIFICATION_FORMAT"
+
+	kubernetes_ensure_resource secret/nova-cell1-rabbitmq
+	NOVA_RABBITMQ_USERNAME=$(get_data_from_secret nova-cell1-rabbitmq openstack username)
+	NOVA_RABBITMQ_PASSWORD=$(get_data_from_secret nova-cell1-rabbitmq openstack password)
+	iniset $NOVA_CONF DEFAULT transport_url "rabbit://$NOVA_RABBITMQ_USERNAME:$NOVA_RABBITMQ_PASSWORD@rabbitmq-nova-cell1.openstack.svc.cluster.local:5672/"
+
+	iniset $NOVA_CONF DEFAULT osapi_compute_workers "$API_WORKERS"
+	iniset $NOVA_CONF DEFAULT metadata_workers "$API_WORKERS"
+	# don't let the conductor get out of control now that we're using a pure python db driver
+	iniset $NOVA_CONF conductor workers "$API_WORKERS"
+
+	if is_service_enabled tls-proxy; then
+		iniset $NOVA_CONF DEFAULT glance_protocol https
+		iniset $NOVA_CONF oslo_middleware enable_proxy_headers_parsing True
+	fi
+
+	iniset $NOVA_CONF DEFAULT graceful_shutdown_timeout "$SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT"
+
+	if [ "$NOVA_USE_SERVICE_TOKEN" == "True" ]; then
+		init_nova_service_user_conf
+	fi
+
+	if is_service_enabled n-cond; then
+		for i in $(seq 1 $NOVA_NUM_CELLS); do
+			local conf
+			local vhost
+			conf=$(conductor_conf $i)
+			vhost="nova_cell${i}"
+			# clean old conductor conf
+			rm -f $conf
+			iniset $conf database connection `database_connection_url nova_cell${i}`
+			iniset $conf conductor workers "$API_WORKERS"
+			iniset $conf DEFAULT debug "$ENABLE_DEBUG_LOG_LEVEL"
+			# if we have a singleconductor, we don't have per host message queues.
+			if [[ "${CELLSV2_SETUP}" == "singleconductor" ]]; then
+				kubernetes_ensure_resource secret/nova-cell1-rabbitmq
+				NOVA_RABBITMQ_USERNAME=$(get_data_from_secret nova-cell1-rabbitmq openstack username)
+				NOVA_RABBITMQ_PASSWORD=$(get_data_from_secret nova-cell1-rabbitmq openstack password)
+				iniset $NOVA_CONF DEFAULT transport_url "rabbit://$NOVA_RABBITMQ_USERNAME:$NOVA_RABBITMQ_PASSWORD@rabbitmq-nova-cell1.openstack.svc.cluster.local:5672/"
+			else
+				# NOTE(mnaser): Not supported for now and all this code is going away anyways
+				exit 1
+			fi
+			# Format logging
+			setup_logging $conf
+		done
+	fi
+
+	# Console proxy configuration has to go after conductor configuration
+	# because the per cell config file nova_cellN.conf is cleared out as part
+	# of conductor configuration.
+	if [[ "${CELLSV2_SETUP}" == "singleconductor" ]]; then
+		configure_console_proxies
+	else
+		for i in $(seq 1 $NOVA_NUM_CELLS); do
+			local conf
+			local offset
+			conf=$(conductor_conf $i)
+			offset=$((i - 1))
+			configure_console_proxies $conf $offset
+		done
+	fi
+}

devstack/settings

@@ -23,5 +23,6 @@ source $DEST/openstack-operator/devstack/lib/glance
 source $DEST/openstack-operator/devstack/lib/horizon
 source $DEST/openstack-operator/devstack/lib/keystone
 source $DEST/openstack-operator/devstack/lib/neutron-legacy
+source $DEST/openstack-operator/devstack/lib/nova
 source $DEST/openstack-operator/devstack/lib/placement
 source $DEST/openstack-operator/devstack/lib/rpc_backend

images/neutron/Dockerfile

@@ -15,6 +15,8 @@
 
 FROM vexxhost/python-builder as builder
 FROM vexxhost/python-base AS neutron-base
+RUN mkdir -p /var/lib/neutron && \
+    chmod 777 -R /var/lib/neutron
 
 FROM neutron-base AS neutron-api
 COPY neutron-api /usr/local/bin/neutron-api
@@ -28,4 +30,16 @@ CMD ["/usr/local/bin/neutron-rpc-server"]
 
 FROM neutron-base AS neutron-openvswitch-agent
 COPY neutron-openvswitch-agent /usr/local/bin/neutron-openvswitch-agent
-CMD ["/usr/local/bin/neutron-openvswitch-agent", "--config-file", "/etc/neutron/neutron.conf", "--config-file", "/etc/neutron/plugins/ml2/ml2_conf.ini"]
+CMD ["/usr/local/bin/neutron-openvswitch-agent", "--config-file", "/etc/neutron/neutron.conf", "--config-file", "/etc/neutron/plugins/ml2/ml2_conf.ini"]
+
+FROM neutron-base AS neutron-l3-agent
+COPY neutron-l3-agent /usr/local/bin/neutron-l3-agent
+CMD ["/usr/local/bin/neutron-l3-agent", "--config-file", "/etc/neutron/neutron.conf", "--config-file", "/etc/neutron/l3_agent.ini", "--config-file", "/etc/neutron/plugins/ml2/ml2_conf.ini"]
+
+FROM neutron-base AS neutron-dhcp-agent
+COPY neutron-dhcp-agent /usr/local/bin/neutron-dhcp-agent
+CMD ["/usr/local/bin/neutron-dhcp-agent", "--config-file", "/etc/neutron/neutron.conf", "--config-file", "/etc/neutron/dhcp_agent.ini", "--config-file", "/etc/neutron/plugins/ml2/ml2_conf.ini"]
+
+FROM neutron-base AS neutron-metadata-agent
+COPY neutron-metadata-agent /usr/local/bin/neutron-metadata-agent
+CMD ["/usr/local/bin/neutron-metadata-agent", "--config-file", "/etc/neutron/neutron.conf", "--config-file", "/etc/neutron/metadata_agent.ini", "--config-file", "/etc/neutron/plugins/ml2/ml2_conf.ini"]

images/neutron/bindep.txt

@@ -2,3 +2,9 @@ gcc [compile]
 libc-dev [compile]
 sudo
 openvswitch-common
+procps
+iptables
+keepalived
+haproxy
+dnsmasq
+radvd

images/neutron/neutron-dhcp-agent

@@ -0,0 +1,29 @@
+#!/usr/local/bin/python
+# Copyright (c) 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pkg_resources
+import re
+import sys
+
+import sentry_sdk
+
+from neutron.cmd.eventlet.agents.dhcp import main
+
+VERSION = pkg_resources.get_distribution("neutron").version
+sentry_sdk.init(release="neutron@%s" % VERSION)
+
+sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+sys.exit(main())

images/neutron/neutron-l3-agent

@@ -0,0 +1,29 @@
+#!/usr/local/bin/python
+# Copyright (c) 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pkg_resources
+import re
+import sys
+
+import sentry_sdk
+
+from neutron.cmd.eventlet.agents.l3 import main
+
+VERSION = pkg_resources.get_distribution("neutron").version
+sentry_sdk.init(release="neutron@%s" % VERSION)
+
+sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+sys.exit(main())

images/neutron/neutron-metadata-agent

@@ -0,0 +1,29 @@
+#!/usr/local/bin/python
+# Copyright (c) 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pkg_resources
+import re
+import sys
+
+import sentry_sdk
+
+from neutron.cmd.eventlet.agents.metadata import main
+
+VERSION = pkg_resources.get_distribution("neutron").version
+sentry_sdk.init(release="neutron@%s" % VERSION)
+
+sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+sys.exit(main())

openstack_operator/neutron.py

@@ -43,6 +43,10 @@ def create_or_resume(spec, **_):
     utils.create_or_update('neutron/daemonset-server.yml.j2', spec=spec)
     utils.create_or_update('neutron/daemonset-openvswitch-agent.yml.j2',
                            spec=spec)
+    utils.create_or_update('neutron/daemonset-l3-agent.yml.j2', spec=spec)
+    utils.create_or_update('neutron/daemonset-dhcp-agent.yml.j2', spec=spec)
+    utils.create_or_update('neutron/daemonset-metadata-agent.yml.j2',
+                           spec=spec)
     utils.create_or_update('neutron/service.yml.j2')
 
     identity.ensure_application_credential(name="neutron")

openstack_operator/nova.py

@@ -0,0 +1,47 @@
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Nova service
+
+This code takes care of doing the operations of the OpenStack Nova API
+service.
+"""
+
+from openstack_operator import utils
+
+MEMCACHED = True
+
+# NOTE(mnaser): Implement dynamic cells
+CELLS = [
+    'cell0',
+    'cell1'
+]
+
+
+def create_or_resume(**_):
+    """Create and re-sync a Nova instance
+
+    This function is called when a new resource is created but also when we
+    start the service up for the first time.
+    """
+
+    for cell in CELLS:
+        # NOTE(mnaser): cell0 does not need a message queue
+        if cell != 'cell0':
+            if not utils.ensure_secret("openstack", "nova-%s-rabbitmq" % cell):
+                utils.create_or_update('nova/secret-rabbitmq.yml.j2',
+                                       component=cell,
+                                       password=utils.generate_password())
+            utils.create_or_update('nova/rabbitmq.yml.j2', component=cell)

openstack_operator/operator.py

@@ -36,6 +36,7 @@ from openstack_operator import horizon
 from openstack_operator import keystone
 from openstack_operator import libvirtd_exporter
 from openstack_operator import magnum
+from openstack_operator import nova
 from openstack_operator import neutron
 from openstack_operator import placement
 from openstack_operator import utils
@@ -100,6 +101,9 @@ def deploy(name, namespace, new, **_):
     if "neutron" in config:
         spec = set_service_config(config, "neutron")
         neutron.create_or_resume(spec)
+    if "nova" in config:
+        spec = set_service_config(config, "nova")
+        nova.create_or_resume()
     if "horizon" in config:
         spec = set_service_config(config, "horizon")
         horizon.create_or_resume("horizon", spec)

openstack_operator/templates/neutron/daemonset-dhcp-agent.yml.j2

@@ -0,0 +1,83 @@
+---
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: neutron-dhcp-agent
+  namespace: openstack
+  labels:
+    {{ labels("neutron", component="dhcp-agent") | indent(4) }}
+spec:
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      {{ labels("neutron", component="dhcp-agent") | indent(6) }}
+  template:
+    metadata:
+      labels:
+        {{ labels("neutron", component="dhcp-agent") | indent(8) }}
+    spec:
+      automountServiceAccountToken: false
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      containers:
+      - name: agent
+        image: vexxhost/neutron-dhcp-agent:latest
+        imagePullPolicy: Always
+        env:
+        {% if 'sentryDSN' in spec %}
+        - name: SENTRY_DSN
+          value: {{ spec.sentryDSN }}
+        {% endif %}
+        securityContext:
+          # NOTE(mnaser): We need to revisit this
+          privileged: true
+        volumeMounts:
+        - name: config
+          mountPath: /etc/neutron
+        - name: ml2-config
+          mountPath: /etc/neutron/plugins/ml2
+        - name: host-run-ovs
+          mountPath: /run/openvswitch
+        - name: host-run-netns
+          mountPath: /run/netns
+          mountPropagation: Bidirectional
+      volumes:
+      - name: config
+        secret:
+          secretName: neutron-config
+      - name: ml2-config
+        secret:
+          secretName: neutron-ml2-config
+      - name: host-run-ovs
+        hostPath:
+          path: /run/openvswitch
+      - name: host-run-netns
+        hostPath:
+          path: /run/netns
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+{% if 'hostAliases' in spec %}
+      hostAliases:
+        {{ spec.hostAliases | to_yaml | indent(8) }}
+{% endif %}
+

openstack_operator/templates/neutron/daemonset-l3-agent.yml.j2

@@ -0,0 +1,89 @@
+---
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: neutron-l3-agent
+  namespace: openstack
+  labels:
+    {{ labels("neutron", component="l3-agent") | indent(4) }}
+spec:
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      {{ labels("neutron", component="l3-agent") | indent(6) }}
+  template:
+    metadata:
+      labels:
+        {{ labels("neutron", component="l3-agent") | indent(8) }}
+    spec:
+      automountServiceAccountToken: false
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      containers:
+      - name: agent
+        image: vexxhost/neutron-l3-agent:latest
+        imagePullPolicy: Always
+        env:
+        {% if 'sentryDSN' in spec %}
+        - name: SENTRY_DSN
+          value: {{ spec.sentryDSN }}
+        {% endif %}
+        securityContext:
+          # NOTE(mnaser): We need to revisit this
+          privileged: true
+        volumeMounts:
+        - name: config
+          mountPath: /etc/neutron
+        - name: ml2-config
+          mountPath: /etc/neutron/plugins/ml2
+        - name: state
+          mountPath: /var/lib/neutron
+        - name: host-run-ovs
+          mountPath: /run/openvswitch
+        - name: host-run-netns
+          mountPath: /run/netns
+          mountPropagation: Bidirectional
+      volumes:
+      - name: config
+        secret:
+          secretName: neutron-config
+      - name: ml2-config
+        secret:
+          secretName: neutron-ml2-config
+      - name: state
+        hostPath:
+          path: /var/lib/neutron
+          type: DirectoryOrCreate
+      - name: host-run-ovs
+        hostPath:
+          path: /run/openvswitch
+      - name: host-run-netns
+        hostPath:
+          path: /run/netns
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+{% if 'hostAliases' in spec %}
+      hostAliases:
+        {{ spec.hostAliases | to_yaml | indent(8) }}
+{% endif %}
+

openstack_operator/templates/neutron/daemonset-metadata-agent.yml.j2

@@ -0,0 +1,84 @@
+---
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: neutron-metadata-agent
+  namespace: openstack
+  labels:
+    {{ labels("neutron", component="metadata-agent") | indent(4) }}
+spec:
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      {{ labels("neutron", component="metadata-agent") | indent(6) }}
+  template:
+    metadata:
+      labels:
+        {{ labels("neutron", component="metadata-agent") | indent(8) }}
+    spec:
+      automountServiceAccountToken: false
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      containers:
+      - name: agent
+        image: vexxhost/neutron-metadata-agent:latest
+        imagePullPolicy: Always
+        env:
+        {% if 'sentryDSN' in spec %}
+        - name: SENTRY_DSN
+          value: {{ spec.sentryDSN }}
+        {% endif %}
+        securityContext:
+          # NOTE(mnaser): We need to revisit this
+          privileged: true
+        volumeMounts:
+        - name: config
+          mountPath: /etc/neutron
+        - name: ml2-config
+          mountPath: /etc/neutron/plugins/ml2
+        - name: state
+          mountPath: /var/lib/neutron
+        - name: host-run-netns
+          mountPath: /run/netns
+          mountPropagation: Bidirectional
+      volumes:
+      - name: config
+        secret:
+          secretName: neutron-config
+      - name: ml2-config
+        secret:
+          secretName: neutron-ml2-config
+      - name: state
+        hostPath:
+          path: /var/lib/neutron
+          type: DirectoryOrCreate
+      - name: host-run-netns
+        hostPath:
+          path: /run/netns
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+{% if 'hostAliases' in spec %}
+      hostAliases:
+        {{ spec.hostAliases | to_yaml | indent(8) }}
+{% endif %}
+

openstack_operator/templates/nova/rabbitmq.yml.j2

@@ -0,0 +1,29 @@
+---
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: infrastructure.vexxhost.cloud/v1alpha1
+kind: Rabbitmq
+metadata:
+  name: nova-{{ component }}
+  namespace: openstack
+  labels:
+    {{ labels("nova", component=component) | indent(4) }}
+spec:
+  authSecret: nova-{{ component }}-rabbitmq
+  nodeSelector:
+    node-role.kubernetes.io/master: ""
+  tolerations:
+  - key: node-role.kubernetes.io/master
+    effect: NoSchedule

openstack_operator/templates/nova/secret-rabbitmq.yml.j2

@@ -0,0 +1,25 @@
+---
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+metadata:
+  name: nova-{{ component }}-rabbitmq
+  namespace: openstack
+  labels:
+    {{ labels("neutron", component=component) | indent(4) }}
+stringData:
+  username: nova-{{ component }}
+  password: {{ password }}
+kind: Secret

playbooks/functional/devstack.yaml

@@ -16,12 +16,6 @@
 
 - hosts: all
   tasks:
-    - name: Get Memcached IP address
-      command: kubectl get svc/mcrouter-memcached-devstack -o=jsonpath='{.spec.clusterIP}'
-      register: _memcached_ip
-    - name: Get RabbitMQ IP address
-      command: kubectl get svc/rabbitmq-sample -o=jsonpath='{.spec.clusterIP}'
-      register: _rabbitmq_ip
     # NOTE(mnaser): We need to rewrite the devstack local.conf because it happens
     #               inside pre.yaml right now.  This should all be gone once the
     #               operator can deploy OpenStack entirely by itself.
@@ -48,8 +42,6 @@
         _devstack_localrc: "{{ devstack_localrc }}"
         _devstack_localrc_extra:
           CELLSV2_SETUP: singleconductor
-          MEMCACHE_SERVERS: "{{ _memcached_ip.stdout }}:11211"
-          RABBIT_HOST: "{{ _rabbitmq_ip.stdout }}"
           DATABASE_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}"
           ERROR_ON_CLONE: True
     - name: Re-write local.conf

playbooks/functional/run.yaml

@@ -53,9 +53,5 @@
         helm_release_name: openstack-operator
         helm_chart: ./chart
         helm_values_file: ./chart/test-values.yaml
-  tasks:
-    # TODO(mnaser): Generate all manifests and ensure git is not dirty
-    - include_tasks: tests/memcached.yaml
-    - include_tasks: tests/rabbitmq.yaml
 
 - import_playbook: devstack.yaml

playbooks/functional/tests/memcached.yaml

@@ -1,64 +0,0 @@
----
-# Copyright (c) 2020 VEXXHOST, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Create Memcached custom resource
-  command: kubectl apply -f {{ zuul.project.src_dir }}/config/samples/infrastructure_v1alpha1_memcached.yaml
-
-- name: Wait for all pods to become Ready
-  include_role:
-    name: wait-for-pods
-
-- name: Get all pod IPs for Memcached
-  command: kubectl get pods -l app.kubernetes.io/name=memcached,app.kubernetes.io/instance=devstack -o=jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}'
-  register: _memcached_ips
-  until: _memcached_ips is success
-  retries: 60
-  delay: 5
-  failed_when: |
-    {{ _memcached_ips.stdout_lines | length == 0 }} or
-    {{ "" in _memcached_ips.stdout_lines }}
-
-- name: Send request to Memcached exporter
-  uri:
-    url: "http://{{ item }}:9150/metrics"
-    return_content: yes
-  register: _metrics
-  loop: "{{ _memcached_ips.stdout_lines }}"
-  until: _metrics is success
-  retries: 10
-  delay: 5
-  failed_when: "'memcached_up 1' not in _metrics.content"
-
-- name: Get all pod IPs for Mcrouter
-  command: kubectl get pods -l app.kubernetes.io/name=mcrouter,app.kubernetes.io/instance=memcached-devstack -o=jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}'
-  register: _mcrouter_ips
-  until: _mcrouter_ips is success
-  retries: 60
-  delay: 5
-  failed_when: |
-    {{ _mcrouter_ips.stdout_lines | length == 0 }} or
-    {{ "" in _mcrouter_ips.stdout_lines }}
-
-- name: Send request to Mcrouter exporter
-  uri:
-    url: "http://{{ item }}:9442/metrics"
-    return_content: yes
-  register: _metrics
-  loop: "{{ _mcrouter_ips.stdout_lines }}"
-  until: _metrics is success
-  retries: 10
-  delay: 5
-  failed_when: "'mcrouter_up 1' not in _metrics.content"

playbooks/functional/tests/rabbitmq.yaml

@@ -1,54 +0,0 @@
----
-# Copyright (c) 2020 VEXXHOST, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Create Rabbitmq credential
-  shell: |
-    cat <<EOF | kubectl apply -f-
-    apiVersion: v1
-    metadata:
-      name: rabbitmq-sample
-      namespace: default
-    stringData:
-      username: stackrabbit
-      password: secretrabbit
-    kind: Secret
-    EOF
-
-- name: Create Rabbitmq custom resource
-  command: kubectl apply -f {{ zuul.project.src_dir }}/config/samples/infrastructure_v1alpha1_rabbitmq.yaml
-
-- name: Wait for all pods to become Ready
-  include_role:
-    name: wait-for-pods
-
-- name: Get all pod IPs for Rabbitmq
-  command: kubectl get pods -l app.kubernetes.io/name=rabbitmq,app.kubernetes.io/instance=sample -o=jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}'
-  register: _rabbitmq_ips
-  until: _rabbitmq_ips is success
-  retries: 60
-  delay: 5
-  failed_when: "{{ _rabbitmq_ips.stdout_lines | length == 0 }}"
-
-- name: Send request to Rabbitmq exporter
-  uri:
-    url: "http://{{ item }}:15692/metrics"
-    return_content: yes
-  register: _metrics
-  loop: "{{ _rabbitmq_ips.stdout_lines }}"
-  until: _metrics is success
-  retries: 10
-  delay: 5
-  failed_when: "'rabbitmq_erlang_uptime_seconds' not in _metrics.content"

setup.cfg

@@ -10,6 +10,7 @@ operators =
     keystone = openstack_operator.keystone
     magnum = openstack_operator.magnum
     neutron = openstack_operator.neutron
+    nova = openstack_operator.nova
 
 [files]
 packages =

zuul.d/functional-jobs.yaml

@@ -30,7 +30,7 @@
         c-bak: false
         etcd3: false
         horizon: true
-        rabbit: false
+        rabbit: true
         tls-proxy: false
       devstack_plugins:
         openstack-operator: https://opendev.org/vexxhost/openstack-operator
@@ -43,7 +43,7 @@
       - src/opendev.org/vexxhost
       tox_envlist: all
       tempest_test_regex: (\[.*\bsmoke\b.*\]|(^heat_tempest_plugin.tests.api)|(^tempest_horizon.tests.scenario)|(^barbican_tempest_plugin.tests.api)|(^barbican_tempest_plugin.tests.scenario))
-      tempest_black_regex: (^tempest.scenario.test_network_basic_ops|barbican_tempest_plugin.tests.scenario.(test_image_signing.ImageSigningTest.test_signed_image_upload_boot_failure|test_volume_encryption.VolumeEncryptionTest.test_encrypted_cinder_volumes_cryptsetup))
+      tempest_black_regex: (^tempest.scenario.test_network_basic_ops|barbican_tempest_plugin.tests.scenario.(test_certificate_validation|test_image_signing.ImageSigningTest.test_signed_image_upload_boot_failure|test_volume_encryption.VolumeEncryptionTest.test_encrypted_cinder_volumes_cryptsetup))
       tempest_plugins:
       - barbican-tempest-plugin
       - heat-tempest-plugin