From c3e085b800bb3236a0c68dd74ffa8c2ec2896e53 Mon Sep 17 00:00:00 2001
From: Gage Hugo <gagehugo@gmail.com>
Date: Wed, 11 Sep 2019 11:56:08 -0500
Subject: [PATCH] Add network policy nonvoting checks
This change adds two network policy zuul checks, one for the compute-kit,
and one for cinder/ceph, to test network policy for each OpenStack
service. These checks will be non-voting initially.
The network policy rules for each service will initially allow all
traffic. These ingress/egress rules will be defined in future changes
to only explicitly allow traffic between services that are explicitly
allowed to communicate, other traffic will be denied.
Depends-On: https://review.opendev.org/#/c/685130/
Change-Id: Ide2998ebb2af2832f24ca7abc398a82e4a6d70e3
---
cinder/values_overrides/netpol.yaml | 2 +
glance/values.yaml | 30 +------
glance/values_overrides/netpol.yaml | 35 ++++++++
heat/values.yaml | 34 +-------
heat/values_overrides/netpol.yaml | 39 +++++++++
horizon/values.yaml | 16 +---
horizon/values_overrides/netpol.yaml | 2 +
keystone/values.yaml | 86 ++-----------------
keystone/values_overrides/netpol.yaml | 84 ++++++++++++++++++
neutron/values_overrides/netpol.yaml | 2 +
nova/values.yaml | 16 ----
nova/values_overrides/netpol.yaml | 2 +
.../lockdown-netpol.sh} | 0
tools/deployment/common/openstack-exporter.sh | 36 ++++++++
tools/deployment/common/test-networkpolicy.sh | 48 +++++++----
zuul.d/jobs-openstack-helm.yaml | 59 +++++++++++++
zuul.d/project.yaml | 4 +
17 files changed, 308 insertions(+), 187 deletions(-)
create mode 100644 cinder/values_overrides/netpol.yaml
create mode 100644 glance/values_overrides/netpol.yaml
create mode 100644 heat/values_overrides/netpol.yaml
create mode 100644 horizon/values_overrides/netpol.yaml
create mode 100644 keystone/values_overrides/netpol.yaml
create mode 100644 neutron/values_overrides/netpol.yaml
create mode 100644 nova/values_overrides/netpol.yaml
rename tools/deployment/{developer/common/049-lockdown.sh => common/lockdown-netpol.sh} (100%)
create mode 100755 tools/deployment/common/openstack-exporter.sh
diff --git a/cinder/values_overrides/netpol.yaml b/cinder/values_overrides/netpol.yaml
new file mode 100644
index 0000000000..7a85753209
--- /dev/null
+++ b/cinder/values_overrides/netpol.yaml
@@ -0,0 +1,2 @@
+manifests:
+ network_policy: true
diff --git a/glance/values.yaml b/glance/values.yaml
index 17bdf4dfe2..7e5fab674f 100644
--- a/glance/values.yaml
+++ b/glance/values.yaml
@@ -88,35 +88,7 @@ ceph_client:
network_policy:
glance:
ingress:
- - from:
- - podSelector:
- matchLabels:
- application: glance
- - podSelector:
- matchLabels:
- application: nova
- - podSelector:
- matchLabels:
- application: horizon
- - podSelector:
- matchLabels:
- application: ingress
- - podSelector:
- matchLabels:
- application: heat
- - podSelector:
- matchLabels:
- application: ironic
- - podSelector:
- matchLabels:
- application: cinder
- ports:
- - protocol: TCP
- port: 80
- - protocol: TCP
- port: 9191
- - protocol: TCP
- port: 9292
+ - {}
egress:
- {}
diff --git a/glance/values_overrides/netpol.yaml b/glance/values_overrides/netpol.yaml
new file mode 100644
index 0000000000..4c6afc2b1c
--- /dev/null
+++ b/glance/values_overrides/netpol.yaml
@@ -0,0 +1,35 @@
+manifests:
+ network_policy: true
+#NOTE(gagehugo): Test this whitelist when the netpol gate works
+#network_policy:
+# glance:
+# ingress:
+# - from:
+# - podSelector:
+# matchLabels:
+# application: glance
+# - podSelector:
+# matchLabels:
+# application: nova
+# - podSelector:
+# matchLabels:
+# application: horizon
+# - podSelector:
+# matchLabels:
+# application: ingress
+# - podSelector:
+# matchLabels:
+# application: heat
+# - podSelector:
+# matchLabels:
+# application: ironic
+# - podSelector:
+# matchLabels:
+# application: cinder
+# ports:
+# - protocol: TCP
+# port: 80
+# - protocol: TCP
+# port: 9191
+# - protocol: TCP
+# port: 9292
diff --git a/heat/values.yaml b/heat/values.yaml
index f9ac5e7b76..26bd874220 100644
--- a/heat/values.yaml
+++ b/heat/values.yaml
@@ -1249,39 +1249,9 @@ pod:
network_policy:
heat:
ingress:
- - from:
- - podSelector:
- matchLabels:
- application: heat
- - podSelector:
- matchLabels:
- application: ingress
- - podSelector:
- matchLabels:
- application: horizon
- ports:
- - protocol: TCP
- port: 80
- - protocol: TCP
- port: 8000
- - protocol: TCP
- port: 8003
- - protocol: TCP
- port: 8004
+ - {}
egress:
- - to:
- - podSelector:
- matchLabels:
- application: neutron
- - podSelector:
- matchLabels:
- application: nova
- - podSelector:
- matchLabels:
- application: glance
- - podSelector:
- matchLabels:
- application: cinder
+ - {}
manifests:
configmap_bin: true
diff --git a/heat/values_overrides/netpol.yaml b/heat/values_overrides/netpol.yaml
new file mode 100644
index 0000000000..e16bc97c5e
--- /dev/null
+++ b/heat/values_overrides/netpol.yaml
@@ -0,0 +1,39 @@
+manifests:
+ network_policy: true
+#NOTE(gagehugo): Test these once the netpol gate works
+#network_policy:
+# heat:
+# ingress:
+# - from:
+# - podSelector:
+# matchLabels:
+# application: heat
+# - podSelector:
+# matchLabels:
+# application: ingress
+# - podSelector:
+# matchLabels:
+# application: horizon
+# ports:
+# - protocol: TCP
+# port: 80
+# - protocol: TCP
+# port: 8000
+# - protocol: TCP
+# port: 8003
+# - protocol: TCP
+# port: 8004
+# egress:
+# - to:
+# - podSelector:
+# matchLabels:
+# application: neutron
+# - podSelector:
+# matchLabels:
+# application: nova
+# - podSelector:
+# matchLabels:
+# application: glance
+# - podSelector:
+# matchLabels:
+# application: cinder
diff --git a/horizon/values.yaml b/horizon/values.yaml
index c16b865997..7a66e051bb 100644
--- a/horizon/values.yaml
+++ b/horizon/values.yaml
@@ -2237,19 +2237,9 @@ endpoints:
network_policy:
horizon:
ingress:
- - from:
- - podSelector:
- matchLabels:
- application: horizon
- - podSelector:
- matchLabels:
- application: ingress
- - namespaceSelector:
- matchLabels:
- name: kube-system
- - ports:
- - protocol: TCP
- port: 80
+ - {}
+ egress:
+ - {}
manifests:
configmap_bin: true
diff --git a/horizon/values_overrides/netpol.yaml b/horizon/values_overrides/netpol.yaml
new file mode 100644
index 0000000000..7a85753209
--- /dev/null
+++ b/horizon/values_overrides/netpol.yaml
@@ -0,0 +1,2 @@
+manifests:
+ network_policy: true
diff --git a/keystone/values.yaml b/keystone/values.yaml
index 6ff6daef67..c5134bfcbe 100644
--- a/keystone/values.yaml
+++ b/keystone/values.yaml
@@ -384,86 +384,12 @@ jobs:
failed: 1
network_policy:
- keystone:
- ingress:
- - from:
- - podSelector:
- matchLabels:
- application: ceph
- - podSelector:
- matchLabels:
- application: ingress
- - podSelector:
- matchLabels:
- application: keystone
- - podSelector:
- matchLabels:
- application: heat
- - podSelector:
- matchLabels:
- application: glance
- - podSelector:
- matchLabels:
- application: cinder
- - podSelector:
- matchLabels:
- application: congress
- - podSelector:
- matchLabels:
- application: barbican
- - podSelector:
- matchLabels:
- application: ceilometer
- - podSelector:
- matchLabels:
- application: horizon
- - podSelector:
- matchLabels:
- application: ironic
- - podSelector:
- matchLabels:
- application: magnum
- - podSelector:
- matchLabels:
- application: mistral
- - podSelector:
- matchLabels:
- application: nova
- - podSelector:
- matchLabels:
- application: neutron
- - podSelector:
- matchLabels:
- application: senlin
- - podSelector:
- matchLabels:
- application: placement
- - podSelector:
- matchLabels:
- application: prometheus-openstack-exporter
- ports:
- - protocol: TCP
- port: 80
- - protocol: TCP
- port: 443
- - protocol: TCP
- port: 5000
- - protocol: TCP
- port: 35357
- egress:
- - to:
- - namespaceSelector:
- matchLabels:
- name: ceph
- - to:
- - podSelector:
- matchLabels:
- application: ceph
- - ports:
- - port: 53
- protocol: UDP
- - port: 53
- protocol: TCP
+ keystone:
+ ingress:
+ - {}
+ egress:
+ - {}
+
conf:
security: |
#
diff --git a/keystone/values_overrides/netpol.yaml b/keystone/values_overrides/netpol.yaml
new file mode 100644
index 0000000000..d80d31bed2
--- /dev/null
+++ b/keystone/values_overrides/netpol.yaml
@@ -0,0 +1,84 @@
+manifests:
+ network_policy: true
+#NOTE(gagehugo): Test the below whitelist after netpol gate works
+#network_policy:
+# keystone:
+# ingress:
+# - from:
+# - podSelector:
+# matchLabels:
+# application: ceph
+# - podSelector:
+# matchLabels:
+# application: ingress
+# - podSelector:
+# matchLabels:
+# application: keystone
+# - podSelector:
+# matchLabels:
+# application: heat
+# - podSelector:
+# matchLabels:
+# application: glance
+# - podSelector:
+# matchLabels:
+# application: cinder
+# - podSelector:
+# matchLabels:
+# application: congress
+# - podSelector:
+# matchLabels:
+# application: barbican
+# - podSelector:
+# matchLabels:
+# application: ceilometer
+# - podSelector:
+# matchLabels:
+# application: horizon
+# - podSelector:
+# matchLabels:
+# application: ironic
+# - podSelector:
+# matchLabels:
+# application: magnum
+# - podSelector:
+# matchLabels:
+# application: mistral
+# - podSelector:
+# matchLabels:
+# application: nova
+# - podSelector:
+# matchLabels:
+# application: neutron
+# - podSelector:
+# matchLabels:
+# application: senlin
+# - podSelector:
+# matchLabels:
+# application: placement
+# - podSelector:
+# matchLabels:
+# application: prometheus-openstack-exporter
+# ports:
+# - protocol: TCP
+# port: 80
+# - protocol: TCP
+# port: 443
+# - protocol: TCP
+# port: 5000
+# - protocol: TCP
+# port: 35357
+# egress:
+# - to:
+# - namespaceSelector:
+# matchLabels:
+# name: ceph
+# - to:
+# - podSelector:
+# matchLabels:
+# application: ceph
+# - ports:
+# - port: 53
+# protocol: UDP
+# - port: 53
+# protocol: TCP
diff --git a/neutron/values_overrides/netpol.yaml b/neutron/values_overrides/netpol.yaml
new file mode 100644
index 0000000000..7a85753209
--- /dev/null
+++ b/neutron/values_overrides/netpol.yaml
@@ -0,0 +1,2 @@
+manifests:
+ network_policy: true
diff --git a/nova/values.yaml b/nova/values.yaml
index 29512caaac..124e4ca15d 100644
--- a/nova/values.yaml
+++ b/nova/values.yaml
@@ -2489,22 +2489,6 @@ network_policy:
- {}
egress:
- {}
- - to:
- - podSelector:
- matchLabels:
- application: ceph
- - podSelector:
- matchLabels:
- application: ingress
- - podSelector:
- matchLabels:
- application: openvswitch
- - podSelector:
- matchLabels:
- application: libvirt
- - podSelector:
- matchLabels:
- application: cinder
placement:
# TODO(lamt): Need to tighten this ingress for security.
ingress:
diff --git a/nova/values_overrides/netpol.yaml b/nova/values_overrides/netpol.yaml
new file mode 100644
index 0000000000..7a85753209
--- /dev/null
+++ b/nova/values_overrides/netpol.yaml
@@ -0,0 +1,2 @@
+manifests:
+ network_policy: true
diff --git a/tools/deployment/developer/common/049-lockdown.sh b/tools/deployment/common/lockdown-netpol.sh
similarity index 100%
rename from tools/deployment/developer/common/049-lockdown.sh
rename to tools/deployment/common/lockdown-netpol.sh
diff --git a/tools/deployment/common/openstack-exporter.sh b/tools/deployment/common/openstack-exporter.sh
new file mode 100755
index 0000000000..8c6d07a550
--- /dev/null
+++ b/tools/deployment/common/openstack-exporter.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Copyright 2019 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+set -xe
+
+#NOTE: Get the over-rides to use
+export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${OSH_INFRA_PATH:="../openstack-helm-infra"}"}"
+: ${OSH_EXTRA_HELM_ARGS_OSEXPORTER:="$(./tools/deployment/common/get-values-overrides.sh prometheus-openstack-exporter)"}
+
+#NOTE: Lint and package chart
+make -C ${HELM_CHART_ROOT_PATH} prometheus-openstack-exporter
+
+: ${OSH_EXTRA_HELM_ARGS:=""}
+helm upgrade --install prometheus-openstack-exporter ${HELM_CHART_ROOT_PATH}/prometheus-openstack-exporter \
+ --namespace=openstack \
+ ${OSH_EXTRA_HELM_ARGS} \
+ ${OSH_EXTRA_HELM_ARGS_OSEXPORTER}
+
+#NOTE: Wait for deploy
+./tools/deployment/common/wait-for-pods.sh openstack
+
+#NOTE: Validate Deployment info
+helm status prometheus-openstack-exporter
diff --git a/tools/deployment/common/test-networkpolicy.sh b/tools/deployment/common/test-networkpolicy.sh
index 04e7c337fa..b7f5db7759 100755
--- a/tools/deployment/common/test-networkpolicy.sh
+++ b/tools/deployment/common/test-networkpolicy.sh
@@ -15,14 +15,15 @@
# under the License.
set -xe
-# test_netpol(namespace, component, target_host, expected_result{fail,success})
+# test_netpol(namespace, application, component, target_host, expected_result{fail,success})
function test_netpol {
NS=$1
- COMPONENT=$2
- HOST=$3
- STATUS=$4
- echo Testing connection from $COMPONENT to host $HOST with namespace $NS
- POD=$(kubectl -n $NS get pod | grep $COMPONENT | grep Running | awk '{print $1}')
+ APP=$2
+ COMPONENT=$3
+ HOST=$4
+ STATUS=$5
+ echo Testing connection from $APP - $COMPONENT to host $HOST with namespace $NS
+ POD=$(kubectl -n $NS get pod -l application=$APP,component=$COMPONENT | grep Running | cut -f 1 -d " " | head -n 1)
PID=$(sudo docker inspect --format '{{ .State.Pid }}' $(kubectl get pods --namespace $NS $POD -o jsonpath='{.status.containerStatuses[0].containerID}' | cut -c 10-21))
if [ "x${STATUS}" == "xfail" ]; then
if ! sudo nsenter -t $PID -n wget --spider --timeout=5 --tries=1 $HOST ; then
@@ -34,17 +35,30 @@ function test_netpol {
sudo nsenter -t $PID -n wget --spider --timeout=5 --tries=1 $HOST
fi
}
+
+#NOTE(gagehugo): Enable the negative tests once the services policy is defined
+
+# General Netpol Tests
# Doing negative tests
-test_netpol openstack keystone-api heat-api.openstack.svc.cluster.local fail
-test_netpol openstack keystone-api glance-api.openstack.svc.cluster.local fail
-test_netpol openstack mariadb-server rabbitmq.openstack.svc.cluster.local:5672 fail
-test_netpol openstack rabbitmq-rabbitmq memcached.openstack.svc.cluster.local:11211 fail
-test_netpol openstack memcached mariadb.openstack.svc.cluster.local:3306 fail
-
+#test_netpol openstack mariadb server rabbitmq.openstack.svc.cluster.local:5672 fail
+#test_netpol openstack rabbitmq-rabbitmq server memcached.openstack.svc.cluster.local:11211 fail
+#test_netpol openstack memcached server mariadb.openstack.svc.cluster.local:3306 fail
# Doing positive tests
-test_netpol openstack keystone-api mariadb.openstack.svc.cluster.local:3306 success
-test_netpol openstack keystone-api rabbitmq.openstack.svc.cluster.local:5672 success
-test_netpol openstack heat-api mariadb.openstack.svc.cluster.local:3306 success
-test_netpol openstack glance-api mariadb.openstack.svc.cluster.local:3306 success
+test_netpol openstack keystone api mariadb.openstack.svc.cluster.local:3306 success
+test_netpol openstack keystone api rabbitmq.openstack.svc.cluster.local:5672 success
-echo Test successfully
+if kubectl -n openstack get pod -l application=cinder | grep Running ; then
+# Negative Cinder Tests
+ #test_netpol openstack keystone api cinder-api.openstack.svc.cluster.local fail
+# Positive Cinder Tests
+ test_netpol openstack cinder api rabbitmq.openstack.svc.cluster.local:5672 success
+else
+# Negative Compute-Kit Tests
+ #test_netpol openstack keystone api heat-api.openstack.svc.cluster.local fail
+ #test_netpol openstack keystone api glance-api.openstack.svc.cluster.local fail
+# Positive Compute-Kit Tests
+ test_netpol openstack heat api mariadb.openstack.svc.cluster.local:3306 success
+ test_netpol openstack glance api mariadb.openstack.svc.cluster.local:3306 success
+fi
+
+echo Test Success
diff --git a/zuul.d/jobs-openstack-helm.yaml b/zuul.d/jobs-openstack-helm.yaml
index 94a1122aa0..ed21a9cdc7 100644
--- a/zuul.d/jobs-openstack-helm.yaml
+++ b/zuul.d/jobs-openstack-helm.yaml
@@ -266,6 +266,65 @@
- ./tools/deployment/developer/common/170-setup-gateway.sh
- ./tools/deployment/developer/common/900-use-it.sh
+- job:
+ name: openstack-helm-netpol-compute-kit
+ parent: openstack-helm-chart-deploy
+ timeout: 7200
+ run: tools/gate/playbooks/osh-gate-runner.yaml
+ vars:
+ osh_params:
+ openstack_release: ocata
+ container_distro_name: ubuntu
+ container_distro_version: xenial
+ feature_gates: netpol
+ gate_scripts:
+ - ./tools/deployment/common/install-packages.sh
+ - ./tools/deployment/common/deploy-k8s.sh
+ - ./tools/deployment/common/setup-client.sh
+ - ./tools/deployment/component/common/ingress.sh
+ - ./tools/deployment/common/lockdown-netpol.sh
+ - ./tools/deployment/component/common/mariadb.sh
+ - ./tools/deployment/component/common/memcached.sh
+ - ./tools/deployment/component/common/rabbitmq.sh
+ - ./tools/deployment/component/nfs-provisioner/nfs-provisioner.sh
+ - ./tools/deployment/component/keystone/keystone.sh
+ - ./tools/deployment/component/heat/heat.sh
+ - ./tools/deployment/component/glance/glance.sh
+ - ./tools/deployment/component/compute-kit/openvswitch.sh
+ - ./tools/deployment/component/compute-kit/libvirt.sh
+ - ./tools/deployment/component/compute-kit/compute-kit.sh
+ - ./tools/deployment/developer/common/170-setup-gateway.sh
+ - ./tools/deployment/common/openstack-exporter.sh
+ - ./tools/deployment/developer/common/900-use-it.sh
+ - ./tools/deployment/common/test-networkpolicy.sh
+
+- job:
+ name: openstack-helm-netpol-cinder
+ parent: openstack-helm-chart-deploy
+ timeout: 7200
+ run: tools/gate/playbooks/osh-gate-runner.yaml
+ vars:
+ osh_params:
+ openstack_release: ocata
+ container_distro_name: ubuntu
+ container_distro_version: xenial
+ feature_gates: netpol
+ gate_scripts:
+ - ./tools/deployment/common/install-packages.sh
+ - ./tools/deployment/common/deploy-k8s.sh
+ - ./tools/deployment/common/setup-client.sh
+ - ./tools/deployment/component/ceph/ceph.sh
+ - ./tools/deployment/component/ceph/ceph-ns-activate.sh
+ - ./tools/deployment/common/lockdown-netpol.sh
+ - ./tools/deployment/component/common/ingress.sh
+ - ./tools/deployment/component/common/mariadb.sh
+ - ./tools/deployment/component/common/memcached.sh
+ - ./tools/deployment/component/common/rabbitmq.sh
+ - ./tools/deployment/component/keystone/keystone.sh
+ - ./tools/deployment/component/cinder/cinder.sh
+ - ./tools/deployment/common/openstack-exporter.sh
+ - ./tools/deployment/common/test-networkpolicy.sh
+
- job:
name: openstack-helm-multinode-temp
parent: openstack-helm-functional-temp
diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
index a570a3bd82..6d99f44dff 100644
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@@ -42,6 +42,10 @@
- openstack-helm-horizon
- openstack-helm-apparmor:
voting: false
+ - openstack-helm-netpol-compute-kit:
+ voting: false
+ - openstack-helm-netpol-cinder:
+ voting: false
gate:
jobs:
- openstack-helm-lint