Add TLS OpenStack API endpoints

This patch provides TLS endpoints secured by a self-signed
certificate. Another patch will provide support for trusted CA-signed
certificates.

A new config.tls.generate-cert option is added that defaults to true.
When true, a self-signed certificate will be generated and OpenStack
API endpoints will be configured to use TLS with that self-signed
certificate. The following config options are added:

snap get microstack config.tls.generate-self-signed
snap get microstack config.tls.cacert-path
snap get microstack config.tls.cert-path
snap get microstack config.tls.key-path

Users can provide their own self-signed certificate by setting
generate-self-signed to false and storing their own certificates/key
at the paths specified by cacert-path, cert-path, and key-path.
'snap set' can also be used to change the cert/key file names.

If using clustering, the certificates/key will be copied from the
control node to the compute nodes. The config for cacert-path,
cert-path, and key-path will be set to the same values as on the
control node.

Other notable changes:
* The existing generate_selfsigned() function is modified to change
  the subject alternative name to be made up of the hostname and
  optionally an IP. The controller hostname and IP are used when
  generating the certificate for self-signed TLS endpoints. The
  hostname is now used instead of 'microstack.run' when generating
  the clustering certificate.
* This change also aligns logging for nginx and corresponding sites
  and moves all nginx sites to {snap_common}/etc/nginx/sites-enabled.

Change-Id: Iceea3127822404a3275fcf8a221cbedc4b52c217

DEMO.md

@@ -158,11 +158,11 @@ Answer the questions as follows:
 
 <table>
   <tr><td>cloud type:</td> <td><code>openstack</code></td></tr>
-  <tr><td>endpoint:</td> <td><code>http://10.20.20.1:5000/v3</code></td></tr>
+  <tr><td>endpoint:</td> <td><code>https://10.20.20.1:5000/v3</code></td></tr>
   <tr><td>cert path:</td> <td><code>none</code></td></tr>
   <tr><td>auth type:</td> <td><code>userpass</code></td></tr>
   <tr><td>region:</td> <td><code>microstack</code></td></tr>
-  <tr><td>region endpoint:</td> <td><code>http://10.20.20.1:5000/v3</code></td></tr>
+  <tr><td>region endpoint:</td> <td><code>https://10.20.20.1:5000/v3</code></td></tr>
   <tr><td>add another region?:</td> <td><code>N</code></td></tr>
 </table>
 
@@ -182,7 +182,7 @@ images in your microstack cloud. Here's how to set that up.
 
 ```
 mkdir simplestreams
-juju metadata generate-image -d ~/simplestreams -i $IMAGE -s bionic -r microstack -u http://10.20.20.1:5000/v3
+juju metadata generate-image -d ~/simplestreams -i $IMAGE -s bionic -r microstack -u https://10.20.20.1:5000/v3
 ```
 
 (If you don't still have an `IMAGE` variable in your env, you can find

snap-overlay/bin/set-default-config.py

@@ -35,7 +35,7 @@ def _get_default_config():
         'config.network.ext-cidr': '10.20.20.1/24',
         'config.network.security-rules': True,
         'config.network.dashboard-allowed-hosts': '*',
-        'config.network.ports.dashboard': 80,
+        'config.network.ports.dashboard': 443,
         'config.network.ports.mysql': 3306,
         'config.network.ports.rabbit': 5672,
         'config.network.external-bridge-name': 'br-ex',
@@ -75,6 +75,14 @@ def _get_default_config():
         'config.nova.cpu-mode': 'host-model',
         # Do not override cpu-models by default.
         'config.nova.cpu-models': '',
+
+        'config.tls.generate-self-signed': True,
+        'config.tls.cacert-path':
+        f'{snap_common}/etc/ssl/certs/cacert.pem',
+        'config.tls.cert-path':
+        f'{snap_common}/etc/ssl/certs/cert.pem',
+        'config.tls.key-path':
+        f'{snap_common}/etc/ssl/private/key.pem',
     }
 
 

snap-overlay/snap-openstack.yaml

@@ -8,7 +8,6 @@ setup:
     - "{snap_common}/etc/neutron/policy.d"
     - "{snap_common}/etc/neutron/rootwrap.d"
     - "{snap_common}/etc/nginx/sites-enabled"
-    - "{snap_common}/etc/nginx/snap/sites-enabled"
     - "{snap_common}/etc/glance/glance.conf.d"
     - "{snap_common}/etc/placement/placement.conf.d"
     - "{snap_common}/etc/horizon/horizon.conf.d"
@@ -22,6 +21,8 @@ setup:
     - "{snap_common}/etc/cluster/tls"
     - "{snap_common}/etc/cluster/uwsgi/snap"
     - "{snap_common}/etc/rabbitmq"
+    - "{snap_common}/etc/ssl/certs"
+    - "{snap_common}/etc/ssl/private"
     - "{snap_common}/fernet-keys"
     - "{snap_common}/lib"
     - "{snap_common}/lib/images"
@@ -33,24 +34,25 @@ setup:
     - "{snap_common}/etc/iscsi"
     - "{snap_common}/etc/target"
   templates:
-    cluster-nginx.conf.j2: "{snap_common}/etc/nginx/snap/sites-enabled/cluster.conf"
+    cluster-nginx.conf.j2: "{snap_common}/etc/nginx/sites-enabled/cluster.conf"
-    keystone-nginx.conf.j2: "{snap_common}/etc/nginx/snap/sites-enabled/keystone.conf"
+    keystone-nginx.conf.j2: "{snap_common}/etc/nginx/sites-enabled/keystone.conf"
     keystone-snap.conf.j2: "{snap_common}/etc/keystone/keystone.conf.d/keystone-snap.conf"
     neutron-snap.conf.j2: "{snap_common}/etc/neutron/neutron.conf.d/neutron-snap.conf"
     nginx.conf.j2: "{snap_common}/etc/nginx/snap/nginx.conf"
     nova-snap.conf.j2: "{snap_common}/etc/nova/nova.conf.d/nova-snap.conf"
-    nova-nginx.conf.j2: "{snap_common}/etc/nginx/snap/sites-enabled/nova.conf"
+    nova-nginx.conf.j2: "{snap_common}/etc/nginx/sites-enabled/nova.conf"
     glance-snap.conf.j2: "{snap_common}/etc/glance/glance.conf.d/glance-snap.conf"
-    placement-nginx.conf.j2: "{snap_common}/etc/nginx/snap/sites-enabled/placement.conf"
+    glance-nginx.conf.j2: "{snap_common}/etc/nginx/sites-enabled/glance.conf"
+    placement-nginx.conf.j2: "{snap_common}/etc/nginx/sites-enabled/placement.conf"
     placement-snap.conf.j2: "{snap_common}/etc/placement/placement.conf.d/placement-snap.conf"
-    cinder-nginx.conf.j2: "{snap_common}/etc/nginx/snap/sites-enabled/cinder.conf"
+    cinder-nginx.conf.j2: "{snap_common}/etc/nginx/sites-enabled/cinder.conf"
     cinder-snap.conf.j2: "{snap_common}/etc/cinder/cinder.conf.d/cinder-snap.conf"
     cinder.database.conf.j2: "{snap_common}/etc/cinder/cinder.conf.d/database.conf"
     cinder.rabbitmq.conf.j2: "{snap_common}/etc/cinder/cinder.conf.d/rabbitmq.conf"
     cinder.keystone.conf.j2: "{snap_common}/etc/cinder/cinder.conf.d/keystone.conf"
     cinder-rootwrap.conf.j2: "{snap_common}/etc/cinder/rootwrap.conf"
     horizon-snap.conf.j2: "{snap_common}/etc/horizon/horizon.conf.d/horizon-snap.conf"
-    horizon-nginx.conf.j2: "{snap_common}/etc/nginx/snap/sites-enabled/horizon.conf"
+    horizon-nginx.conf.j2: "{snap_common}/etc/nginx/sites-enabled/horizon.conf"
     05_snap_tweaks.j2: "{snap_common}/etc/horizon/local_settings.d/_05_snap_tweaks.py"
     libvirtd.conf.j2: "{snap_common}/etc/libvirt/libvirtd.conf"
     qemu.conf.j2: "{snap_common}/etc/libvirt/qemu.conf"
@@ -62,10 +64,12 @@ setup:
     nova.conf.d.keystone.conf.j2: "{snap_common}/etc/nova/nova.conf.d/keystone.conf"
     nova.conf.d.database.conf.j2: "{snap_common}/etc/nova/nova.conf.d/database.conf"
     nova.conf.d.rabbitmq.conf.j2: "{snap_common}/etc/nova/nova.conf.d/rabbitmq.conf"
+    nova.conf.d.cinder.conf.j2: "{snap_common}/etc/nova/nova.conf.d/cinder.conf"
     nova.conf.d.glance.conf.j2: "{snap_common}/etc/nova/nova.conf.d/glance.conf"
     nova.conf.d.neutron.conf.j2: "{snap_common}/etc/nova/nova.conf.d/neutron.conf"
     nova.conf.d.placement.conf.j2: "{snap_common}/etc/nova/nova.conf.d/placement.conf"
     nova.conf.d.console.conf.j2: "{snap_common}/etc/nova/nova.conf.d/console.conf"
+    nova-nginx.conf.j2: "{snap_common}/etc/nginx/sites-enabled/nova.conf"
     keystone.database.conf.j2: "{snap_common}/etc/keystone/keystone.conf.d/database.conf"
     glance.database.conf.j2: "{snap_common}/etc/glance/glance.conf.d/database.conf"
     placement.conf.d.database.conf.j2: "{snap_common}/etc/placement/placement.conf.d/database.conf"
@@ -75,6 +79,7 @@ setup:
     neutron.database.conf.j2: "{snap_common}/etc/neutron/neutron.conf.d/database.conf"
     neutron.conf.d.rabbitmq.conf.j2: "{snap_common}/etc/neutron/neutron.conf.d/rabbitmq.conf"
     neutron_ovn_metadata_agent.ini.j2: "{snap_common}/etc/neutron/neutron_ovn_metadata_agent.ini"
+    neutron-nginx.conf.j2: "{snap_common}/etc/nginx/sites-enabled/neutron.conf"
     rabbitmq.conf.j2: "{snap_common}/etc/rabbitmq/rabbitmq.config"
     iscsid.conf.j2: "{snap_common}/etc/iscsi/iscsid.conf"
     lvm.conf.j2: "{snap_common}/etc/lvm/lvm.conf"
@@ -84,6 +89,9 @@ setup:
     nrpe.cfg.j2: "{snap_common}/etc/nrpe/nrpe-microstack.cfg"
     filebeat.yaml.j2: "{snap_common}/etc/filebeat/filebeat-microstack.yaml"
   chmod:
+    "{snap_common}/etc/ssl": 0755
+    "{snap_common}/etc/ssl/certs": 0755
+    "{snap_common}/etc/ssl/private": 0700
     "{snap_common}/instances": 0755
     "{snap_common}/etc/microstack.rc": 0644
     "{snap_common}/etc/microstack.json": 0644
@@ -124,6 +132,10 @@ setup:
     virt_type: 'config.nova.virt-type'
     cpu_mode: 'config.nova.cpu-mode'
     cpu_models: 'config.nova.cpu-models'
+    tls_generate_self_signed: 'config.tls.generate-self-signed'
+    tls_cacert_path: 'config.tls.cacert-path'
+    tls_cert_path: 'config.tls.cert-path'
+    tls_key_path: 'config.tls.key-path'
 entry_points:
   keystone-manage:
     binary: "{snap}/bin/keystone-manage"

snap-overlay/templates/05_snap_tweaks.j2

@@ -23,7 +23,7 @@ AVAILABLE_THEMES = [
 
 # Point us at keystone.
 OPENSTACK_HOST = "10.20.20.1"
-OPENSTACK_KEYSTONE_URL = "http://%s:5000/v3" % OPENSTACK_HOST
+OPENSTACK_KEYSTONE_URL = "https://%s:5000/v3" % OPENSTACK_HOST
 OPENSTACK_KEYSTONE_DEFAULT_ROLE = "_member_"
 
 # Turn off external access for now.  (This should be turned on once we
@@ -40,4 +40,13 @@ CACHES = {
 }
 SESSION_ENGINE='django.contrib.sessions.backends.cache'
 
-
+# SSL config
+CSRF_COOKIE_SECURE = True
+SESSION_COOKIE_SECURE = True
+{% if tls_generate_self_signed|lower == "false" %}
+# TODO(coreycb): Can we verify cert if self-signed certs
+# include a ca-cert and cert?
+OPENSTACK_SSL_CACERT = "{{ tls_cacert_path }}"
+{% else %}
+OPENSTACK_SSL_NO_VERIFY = True
+{% endif %}

snap-overlay/templates/cinder-nginx.conf.j2

@@ -1,5 +1,9 @@
 server {
-    listen 8776;
+    listen 8776 ssl;
+    ssl_certificate {{ tls_cert_path }};
+    ssl_certificate_key {{ tls_key_path }};
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
     access_log {{ snap_common }}/log/nginx-access.log;
     error_log {{ snap_common }}/log/nginx-error.log;
     location / {

snap-overlay/templates/cinder.keystone.conf.j2

@@ -2,8 +2,8 @@
 auth_strategy = keystone
 
 [keystone_authtoken]
-auth_uri = http://{{ control_ip }}:5000
+auth_uri = https://{{ control_ip }}:5000/v3
-auth_url = http://{{ control_ip }}:5000
+auth_url = https://{{ control_ip }}:5000/v3
 memcached_servers = {{ compute_ip }}:11211
 auth_type = password
 project_domain_name = default
@@ -11,3 +11,4 @@ user_domain_name = default
 project_name = service
 username = cinder
 password = {{ cinder_password }}
+cafile = {{ tls_cacert_path }}

snap-overlay/templates/glance-nginx.conf.j2

@@ -0,0 +1,18 @@
+server {
+    listen 9292 ssl;
+    ssl_certificate {{ tls_cert_path }};
+    ssl_certificate_key {{ tls_key_path }};
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+    access_log {{ snap_common }}/log/nginx-access.log;
+    error_log {{ snap_common }}/log/nginx-error.log;
+    # Disabled for glance image uploads. The maximum size will be
+    # determined by glance settings.
+    client_max_body_size 0;
+    location / {
+        proxy_set_header X-Forwarded-Host $host:$server_port;
+        proxy_set_header X-Forwarded-Server $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_pass http://127.0.0.1:9282;
+    }
+}

snap-overlay/templates/glance-snap.conf.j2

@@ -3,6 +3,7 @@
 state_path = {{ snap_common }}/lib
 # Log to systemd journal
 use_journal = True
+bind_port = 9282
 
 log_file = {{ snap_common }}/log/glance.log
 debug = {{ logging_debug }}

snap-overlay/templates/glance.conf.d.keystone.conf.j2

@@ -1,6 +1,6 @@
 [keystone_authtoken]
-auth_uri = http://{{ control_ip }}:5000
+auth_uri = https://{{ control_ip }}:5000/v3
-auth_url = http://{{ control_ip }}:5000
+auth_url = https://{{ control_ip }}:5000/v3
 memcached_servers = {{ compute_ip }}:11211
 auth_type = password
 project_domain_name = default
@@ -8,6 +8,7 @@ user_domain_name = default
 project_name = service
 username = glance
 password = {{ glance_password }}
+cafile = {{ tls_cacert_path }}
 
 [paste_deploy]
 flavor = keystone

snap-overlay/templates/horizon-nginx.conf.j2

@@ -1,10 +1,11 @@
-# If the OpenStack service has an API that runs behind uwsgi+nginx, you'll need
-# to define this template. Be sure to update "listen" with the port number and
-# also update "api-name" for the socket.
 server {
-    listen {{ dashboard_port }};
+    listen {{ dashboard_port }} ssl;
-    error_log syslog:server=unix:/dev/log;
+    ssl_certificate {{ tls_cert_path }};
-    access_log syslog:server=unix:/dev/log;
+    ssl_certificate_key {{ tls_key_path }};
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+    access_log {{ snap_common }}/log/nginx-access.log;
+    error_log {{ snap_common }}/log/nginx-error.log;
     location / {
         include {{ snap }}/usr/conf/uwsgi_params;
         uwsgi_param SCRIPT_NAME '';

snap-overlay/templates/keystone-nginx.conf.j2

@@ -1,7 +1,11 @@
 server {
-    listen 5000;
+    listen 5000 ssl;
-    error_log syslog:server=unix:/dev/log;
+    ssl_certificate {{ tls_cert_path }};
-    access_log syslog:server=unix:/dev/log;
+    ssl_certificate_key {{ tls_key_path }};
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+    access_log {{ snap_common }}/log/nginx-access.log;
+    error_log {{ snap_common }}/log/nginx-error.log;
     location / {
         include {{ snap }}/usr/conf/uwsgi_params;
         uwsgi_param SCRIPT_NAME '';

snap-overlay/templates/microstack.json.j2

@@ -13,12 +13,19 @@
                 "version": 3
             }
         },
-        "auth_url": "http://{{ control_ip }}:5000",
+        "auth_url": "https://{{ control_ip }}:5000/v3",
         "endpoint_type": null,
+{% if tls_generate_self_signed|lower == "false" %}
+        "https_cacert": "{{ tls_cacert_path }}",
+        "https_cert": "{{ tls_cert_path }}",
+        "https_key": "{{ tls_key_path }}",
+        "https_insecure": false,
+{% else %}
         "https_cacert": "",
         "https_cert": "",
-        "https_insecure": false,
         "https_key": "",
+        "https_insecure": true,
+{% endif %}
         "profiler_conn_str": null,
         "profiler_hmac_key": null,
         "region_name": ""

snap-overlay/templates/microstack.rc.j2

@@ -3,7 +3,8 @@ export OS_USER_DOMAIN_NAME=default
 export OS_PROJECT_NAME=admin
 export OS_USERNAME=admin
 export OS_PASSWORD={{ keystone_password }}
-export OS_AUTH_URL=http://{{ control_ip }}:5000
+export OS_AUTH_PROTOCOL=https
+export OS_AUTH_URL=https://{{ control_ip }}:5000/v3
 export OS_IDENTITY_API_VERSION=3
 export OS_IMAGE_API_VERSION=2
-
+export OS_CACERT={{ tls_cacert_path }}

snap-overlay/templates/neutron-nginx.conf.j2

@@ -0,0 +1,15 @@
+server {
+    listen 9696 ssl;
+    ssl_certificate {{ tls_cert_path }};
+    ssl_certificate_key {{ tls_key_path }};
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+    access_log {{ snap_common }}/log/nginx-access.log;
+    error_log {{ snap_common }}/log/nginx-error.log;
+    location / {
+        proxy_set_header X-Forwarded-Host $host:$server_port;
+        proxy_set_header X-Forwarded-Server $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_pass http://127.0.0.1:9686;
+    }
+}

snap-overlay/templates/neutron-snap.conf.j2

@@ -3,6 +3,7 @@
 state_path = {{ snap_common }}/lib
 # Log to systemd journal
 use_journal = True
+bind_port = 9686
 
 log_file = {{ snap_common }}/log/neutron.log
 debug = {{ logging_debug }}

snap-overlay/templates/neutron.keystone.conf.j2

@@ -2,8 +2,8 @@
 auth_strategy = keystone
 
 [keystone_authtoken]
-auth_uri = http://{{ control_ip }}:5000
+auth_uri = https://{{ control_ip }}:5000/v3
-auth_url = http://{{ control_ip }}:5000
+auth_url = https://{{ control_ip }}:5000/v3
 memcached_servers = {{ compute_ip }}:11211
 auth_type = password
 project_domain_name = default
@@ -11,3 +11,4 @@ user_domain_name = default
 project_name = service
 username = neutron
 password = {{ neutron_password }}
+cafile = {{ tls_cacert_path }}

snap-overlay/templates/neutron.nova.conf.j2

@@ -3,7 +3,7 @@ notify_nova_on_port_status_changes = True
 notify_nova_on_port_data_changes = True
 
 [nova]
-auth_url = http://{{ control_ip }}:5000
+auth_url = https://{{ control_ip }}:5000/v3
 auth_type = password
 project_domain_name = default
 user_domain_name = default
@@ -11,3 +11,4 @@ region_name = {{ region_name }}
 project_name = service
 username = nova
 password = {{ nova_password }}
+cafile = {{ tls_cacert_path }}

snap-overlay/templates/neutron.placement.conf.j2

@@ -1,5 +1,5 @@
 [placement]
-auth_url = http://{{ control_ip }}:5000
+auth_url = https://{{ control_ip }}:5000/v3
 auth_type = password
 project_domain_name = default
 user_domain_name = default
@@ -7,3 +7,4 @@ region_name = {{ region_name }}
 project_name = service
 username = placement
 password = {{ placement_password }}
+cafile = {{ tls_cacert_path }}

snap-overlay/templates/nginx.conf.j2

@@ -45,5 +45,5 @@ http {
         gzip_disable "msie6";
 
         include {{ snap_common }}/etc/nginx/conf.d/*.conf;
-        include {{ snap_common }}/etc/nginx/snap/sites-enabled/*;
+        include {{ snap_common }}/etc/nginx/sites-enabled/*;
 }

snap-overlay/templates/nova-nginx.conf.j2

@@ -1,10 +1,15 @@
 server {
-    listen 8778;
+    listen 8774 ssl;
+    ssl_certificate {{ tls_cert_path }};
+    ssl_certificate_key {{ tls_key_path }};
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
     access_log {{ snap_common }}/log/nginx-access.log;
     error_log {{ snap_common }}/log/nginx-error.log;
     location / {
-        include {{ snap }}/usr/conf/uwsgi_params;
+        proxy_set_header X-Forwarded-Host $host:$server_port;
-        uwsgi_param SCRIPT_NAME '';
+        proxy_set_header X-Forwarded-Server $host;
-        uwsgi_pass unix://{{ snap_common }}/run/placement-api.sock;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_pass http://127.0.0.1:8764;
     }
 }

snap-overlay/templates/nova-snap.conf.j2

@@ -7,6 +7,8 @@ state_path = {{ snap_common }}/lib
 # Log to systemd journal
 use_journal = True
 
+osapi_compute_listen_port = 8764
+
 # Set a hostname to be an FQDN to avoid issues with port binding for
 # which a hostname of a Nova node must match a hostname of an OVN chassis.
 host = {{ node_fqdn }}

snap-overlay/templates/nova.conf.d.cinder.conf.j2

@@ -0,0 +1,5 @@
+[cinder]
+service_type = volume
+service_name = cinder
+region_name = {{ region_name }}
+cafile = {{ tls_cacert_path }}

snap-overlay/templates/nova.conf.d.glance.conf.j2

@@ -2,3 +2,4 @@
 service_type = image
 service_name = glance
 region_name = {{ region_name }}
+cafile = {{ tls_cacert_path }}

snap-overlay/templates/nova.conf.d.keystone.conf.j2

@@ -1,6 +1,6 @@
 [keystone_authtoken]
-auth_uri = http://{{ control_ip }}:5000
+auth_uri = https://{{ control_ip }}:5000/v3
-auth_url = http://{{ control_ip }}:5000
+auth_url = https://{{ control_ip }}:5000/v3
 memcached_servers = {{ compute_ip }}:11211
 auth_type = password
 project_domain_name = default
@@ -8,6 +8,7 @@ user_domain_name = default
 project_name = service
 username = nova
 password = {{ nova_password }}
+cafile = {{ tls_cacert_path }}
 
 [paste_deploy]
 flavor = keystone

snap-overlay/templates/nova.conf.d.neutron.conf.j2

@@ -1,6 +1,6 @@
 [neutron]
-url = http://{{ control_ip }}:9696
+url = https://{{ control_ip }}:9696
-auth_url = http://{{ control_ip }}:5000
+auth_url = https://{{ control_ip }}:5000/v3
 memcached_servers = {{ compute_ip }}:11211
 auth_type = password
 project_domain_name = default
@@ -11,3 +11,4 @@ username = neutron
 password = {{ neutron_password }}
 service_metadata_proxy = True
 metadata_proxy_shared_secret = {{ ovn_metadata_proxy_shared_secret }}
+cafile = {{ tls_cacert_path }}

snap-overlay/templates/nova.conf.d.placement.conf.j2

@@ -1,6 +1,6 @@
 [placement]
-auth_uri = http://{{ control_ip }}:5000
+auth_uri = https://{{ control_ip }}:5000/v3
-auth_url = http://{{ control_ip }}:5000
+auth_url = https://{{ control_ip }}:5000/v3
 memcached_servers = {{ compute_ip }}:11211
 auth_type = password
 project_domain_name = default
@@ -9,3 +9,4 @@ project_name = service
 username = nova
 password = {{ nova_password }}
 region_name = {{ region_name }}
+cafile = {{ tls_cacert_path }}

snap-overlay/templates/placement-nginx.conf.j2

@@ -1,7 +1,11 @@
 server {
-    listen 8778;
+    listen 8778 ssl;
-    error_log syslog:server=unix:/dev/log;
+    ssl_certificate {{ tls_cert_path }};
-    access_log syslog:server=unix:/dev/log;
+    ssl_certificate_key {{ tls_key_path }};
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+    access_log {{ snap_common }}/log/nginx-access.log;
+    error_log {{ snap_common }}/log/nginx-error.log;
     location / {
         include {{ snap }}/usr/conf/uwsgi_params;
         uwsgi_param SCRIPT_NAME '';

snap-overlay/templates/placement.conf.d.keystone.conf.j2

@@ -1,6 +1,6 @@
 [keystone_authtoken]
-auth_uri = http://{{ control_ip }}:5000
+auth_uri = https://{{ control_ip }}:5000/v3
-auth_url = http://{{ control_ip }}:5000
+auth_url = https://{{ control_ip }}:5000/v3
 memcached_servers = {{ compute_ip }}:11211
 auth_type = password
 project_domain_name = default
@@ -8,6 +8,7 @@ user_domain_name = default
 project_name = service
 username = placement
 password = {{ placement_password }}
+cafile = {{ tls_cacert_path }}
 
 [paste_deploy]
 flavor = keystone

snap/hooks/post-refresh

@@ -22,7 +22,7 @@ fi
 # Add default ports for mysql, rabbit and dashboard services.
 # [2019-11-21] build 171 (beta) -> master
 if [ -z "$(snapctl get config.network.ports.dashboard)" ]; then
-        snapctl set config.network.ports.dashboard=80
+        snapctl set config.network.ports.dashboard=443
 fi
 
 if [ -z "$(snapctl get config.network.ports.mysql)" ]; then

tests/basic-test.sh

@@ -190,7 +190,7 @@ echo "++++++++++++++++++++++++++++++++++++++++++++++++++"
 
 export HORIZON_IP
 if [[ $PREFIX == *"multipass"* ]]; then
-    echo "Opening $HORIZON_IP:80 up to the outside world."
+    echo "Opening $HORIZON_IP:443 up to the outside world."
     cat<<EOF > /tmp/_10_hosts.py
 # Allow all hosts to connect to this machine
 ALLOWED_HOSTS = ['*',]

tests/configure-the-things.sh

@@ -26,9 +26,9 @@ sudo systemctl restart snap.microstack.*
 microstack.openstack user show admin || {
     sudo microstack.keystone-manage bootstrap \
         --bootstrap-password $OS_PASSWORD \
-        --bootstrap-admin-url http://10.20.20.1:5000/v3/ \
+        --bootstrap-admin-url https://10.20.20.1:5000/v3/ \
-        --bootstrap-internal-url http://10.20.20.1:5000/v3/ \
+        --bootstrap-internal-url https://10.20.20.1:5000/v3/ \
-        --bootstrap-public-url http://10.20.20.1:5000/v3/ \
+        --bootstrap-public-url https://10.20.20.1:5000/v3/ \
         --bootstrap-region-id microstack
 }
 

tests/framework.py

@@ -434,7 +434,7 @@ class Framework(unittest.TestCase):
             'microstack',
             'config.credentials.keystone-password'
         ]).decode('utf-8')
-        self.driver.get(f'http://{control_ip}:{dashboard_port}/')
+        self.driver.get(f'https://{control_ip}:{dashboard_port}/')
         # Login to horizon!
         self.driver.find_element(By.ID, "id_username").click()
         self.driver.find_element(By.ID, "id_username").send_keys("admin")

tools/cluster/cluster/add_compute.py

@@ -27,16 +27,26 @@ VALIDITY_PERIOD = relativedelta(minutes=20)
 def _create_credential():
     project_name = 'service'
     domain_name = 'default'
-    # TODO: add support for TLS-terminated Keystone once this is supported.
     auth = v3.password.Password(
-        auth_url="http://localhost:5000/v3",
+        auth_url="https://localhost:5000/v3",
         username='nova',
         password=config_get('config.credentials.nova-password'),
         user_domain_name=domain_name,
         project_domain_name=domain_name,
         project_name=project_name
     )
-    sess = session.Session(auth=auth)
+    if config_get('config.tls.generate-self-signed'):
+        # TODO(coreycb): Can we verify cert if self-signed certs
+        # include a ca-cert and cert?
+        sess = session.Session(
+            auth=auth,
+            verify=False,
+        )
+    else:
+        sess = session.Session(
+            auth=auth,
+            verify=config_get('config.tls.cacert-path'),
+        )
     keystone_client = client.Client(session=sess)
 
     # Only allow this credential to list the Keystone catalog. After it

tools/cluster/cluster/client.py

@@ -98,6 +98,19 @@ def join():
     control_ip = response_dict['config']['network']['control-ip']
     shell.config_set(**{'config.network.control-ip': control_ip})
 
+    # Write controller's TLS certificate data to compute node
+    tls_path_map = {
+        'cacert-path': 'tls_cacert',
+        'cert-path': 'tls_cert',
+        'key-path': 'tls_key',
+    }
+    for tls_config, tls_file in tls_path_map.items():
+        tls_path = response_dict['config']['tls'][tls_config]
+        shell.config_set(**{'config.tls.{}'.format(tls_config): tls_path})
+        with open(tls_path, "w") as f:
+            f.write(response_dict[tls_file])
+    shell.config_set(**{'config.tls.generate-self-signed': False})
+
 
 if __name__ == '__main__':
     join()

tools/cluster/cluster/daemon.py

@@ -10,6 +10,7 @@ from werkzeug.exceptions import BadRequest
 
 
 from cluster.shell import check_output
+from cluster.shell import config_get
 
 from keystoneauth1.identity import v3
 from keystoneauth1 import session
@@ -153,8 +154,20 @@ def handle_unexpected_error(error):
 def join_info():
     """Generate the configuration information to return to a client."""
     # TODO: be selective about what we return. For now, we just get everything.
+    info = {}
     config = json.loads(check_output('snapctl', 'get', 'config'))
-    info = {'config': config}
+    info['config'] = config
+
+    # Add the controller's TLS certificate data
+    tls_path_map = {
+        'cacert-path': 'tls_cacert',
+        'cert-path': 'tls_cert',
+        'key-path': 'tls_key',
+    }
+    for tls_config, tls_file in tls_path_map.items():
+        with open(config_get('config.tls.{}'.format(tls_config)), "r") as f:
+            info[tls_file] = f.read()
+
     return info
 
 
@@ -210,8 +223,7 @@ def join():
                          ' authentication data in the request.')
             return MissingAuthDataInRequest()
 
-        # TODO: handle https here when TLS termination support is added.
+        keystone_base_url = 'https://localhost:5000/v3'
-        keystone_base_url = 'http://localhost:5000/v3'
 
         # In an unlikely event of failing to construct an auth object
         # treat it as if invalid data got passed in terms of responding
@@ -231,7 +243,18 @@ def join():
         try:
             # Use the auth object with the app credential to create a session
             # which the Keystone client will use.
-            sess = session.Session(auth=auth)
+            if config_get('config.tls.generate-self-signed'):
+                # TODO(coreycb): Can we verify cert if self-signed certs
+                # include a ca-cert and cert?
+                sess = session.Session(
+                    auth=auth,
+                    verify=False,
+                )
+            else:
+                sess = session.Session(
+                    auth=auth,
+                    verify=config_get('config.tls.cacert-path'),
+                )
         except Exception:
             logger.exception('An exception has occurred while trying to build'
                              ' a Session object with auth data'

tools/init/init/main.py

@@ -182,6 +182,7 @@ def init() -> None:
         # The following are not yet implemented:
         # questions.VmSwappiness(),
         # questions.FileHandleLimits(),
+        questions.TlsCertificates(),
         questions.DashboardAccess(),
         questions.RabbitMq(),
         questions.DatabaseSetup(),

tools/init/init/questions/__init__.py

@@ -26,14 +26,16 @@ limitations under the License.
 import json
 from time import sleep
 from os import path
+from pathlib import Path
+from shutil import copyfile
 
 from init import shell
 from init.shell import (check, call, check_output, sql, nc_wait, log_wait,
                         restart, download, disable, enable)
 from init.config import Env, log
-from init import cluster_tls
+from init import tls
 from init.questions.question import Question
-from init.questions import clustering, network, uninstall  # noqa F401
+from init.questions import clustering, network, tls, uninstall  # noqa F401
 
 
 _env = Env().get_env()
@@ -102,7 +104,13 @@ class Clustering(Question):
                 'config.services.hypervisor': 'true',
             })
             # Generate a self-signed certificate for the clustering service.
-            cluster_tls.generate_selfsigned()
+            cert_path, key_path = (
+                Path(shell.config_get('config.cluster.tls-cert-path')),
+                Path(shell.config_get('config.cluster.tls-key-path')),
+            )
+            tls.generate_self_signed(
+                cert_path, key_path,
+                fingerprint_config='config.cluster.fingerprint')
 
         # Write templates
         check('snap-openstack', 'setup')
@@ -288,6 +296,37 @@ class DashboardAccess(ConfigQuestion):
             hosts=answer))
 
 
+class TlsCertificates(Question):
+
+    _type = 'boolean'
+    _question = 'Do you wish to generate a self-signed certificate for TLS?'
+    config_key = 'config.tls.generate-self-signed'
+
+    def yes(self, answer: str) -> None:
+        role = shell.config_get('config.cluster.role')
+
+        if role == 'control':
+            log.info('Generating TLS Certificate and Key')
+            cert_path, key_path = (
+                Path(shell.config_get('config.tls.cert-path')),
+                Path(shell.config_get('config.tls.key-path')),
+            )
+            tls.generate_self_signed(cert_path, key_path,
+                                     ip=_env['control_ip'])
+            copyfile(Path(shell.config_get('config.tls.cert-path')),
+                     Path(shell.config_get('config.tls.cacert-path')))
+            restart('nginx')
+
+        elif role == 'compute':
+            log.warning('TLS certificate generation can only be performed on '
+                        'control node')
+
+    def no(self, answer: str):
+        log.info('TLS certificates must be provided: config.tls.cacert-path, '
+                 'config.tls.cert-path, and config.tls.key-path.')
+        restart('nginx')
+
+
 class RabbitMq(Question):
     """Wait for Rabbit to start, then setup permissions."""
 
@@ -362,7 +401,7 @@ class DatabaseSetup(Question):
         if call('openstack', 'user', 'show', 'admin'):
             return
 
-        bootstrap_url = 'http://{control_ip}:5000/v3/'.format(**_env)
+        bootstrap_url = 'https://{control_ip}:5000/v3/'.format(**_env)
 
         check('snap-openstack', 'launch', 'keystone-manage', 'bootstrap',
               '--bootstrap-password', _env['keystone_password'],
@@ -401,7 +440,7 @@ class DatabaseSetup(Question):
         log.info('Creating service project ...')
         if not call('openstack', 'project', 'show', 'service'):
             check('openstack', 'project', 'create', '--domain',
-                  'default', '--description', 'Service Project',
+                  'default', '--description', '"Service Project"',
                   'service')
 
         log.info('Keystone configured!')
@@ -536,7 +575,7 @@ class PlacementSetup(Question):
             for endpoint in ['public', 'internal', 'admin']:
                 call('openstack', 'endpoint', 'create', '--region',
                      'microstack', 'placement', endpoint,
-                     'http://{control_ip}:8778'.format(**_env))
+                     'https://{control_ip}:8778'.format(**_env))
 
         log.info('Running Placement DB migrations...')
         check('snap-openstack', 'launch', 'placement-manage', 'db', 'sync')
@@ -612,6 +651,7 @@ class NovaControlPlane(Question):
 
         enable('nova-api')
         restart('nova-compute')
+        restart('nginx')
 
         for service in [
                 'nova-api-metadata',
@@ -630,7 +670,7 @@ class NovaControlPlane(Question):
             for endpoint in ['public', 'internal', 'admin']:
                 call('openstack', 'endpoint', 'create', '--region',
                      'microstack', 'compute', endpoint,
-                     'http://{control_ip}:8774/v2.1'.format(**_env))
+                     'https://{control_ip}:8774/v2.1'.format(**_env))
 
         log.info('Creating default flavors...')
 
@@ -682,7 +722,7 @@ class CinderSetup(Question):
                     check(
                             'openstack', 'endpoint', 'create', '--region',
                             'microstack', f'volume{api_version}', endpoint,
-                            f'http://{control_ip}:8776/{api_version}/'
+                            f'https://{control_ip}:8776/{api_version}/'
                             '$(project_id)s'
                     )
         log.info('Running Cinder DB migrations...')
@@ -754,12 +794,13 @@ class NeutronControlPlane(Question):
             for endpoint in ['public', 'internal', 'admin']:
                 call('openstack', 'endpoint', 'create', '--region',
                      'microstack', 'network', endpoint,
-                     'http://{control_ip}:9696'.format(**_env))
+                     'https://{control_ip}:9696'.format(**_env))
 
         check('snap-openstack', 'launch', 'neutron-db-manage', 'upgrade',
               'head')
         enable('neutron-api')
         enable('neutron-ovn-metadata-agent')
+        restart('nginx')
 
         nc_wait(_env['control_ip'], '9696')
 
@@ -864,10 +905,11 @@ class GlanceSetup(Question):
             for endpoint in ['internal', 'admin', 'public']:
                 check('openstack', 'endpoint', 'create', '--region',
                       'microstack', 'image', endpoint,
-                      'http://{compute_ip}:9292'.format(**_env))
+                      'https://{compute_ip}:9292'.format(**_env))
 
         check('snap-openstack', 'launch', 'glance-manage', 'db_sync')
         enable('glance-api')
+        restart('nginx')
 
         nc_wait(_env['compute_ip'], '9292')
 

tools/init/init/tls.py

@@ -1,9 +1,12 @@
 #!/usr/bin/env python3
 
-from pathlib import Path
+
+from init.shell import check
 
 from datetime import datetime
 from dateutil.relativedelta import relativedelta
+import ipaddress
+import socket
 
 from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.backends import default_backend
@@ -15,7 +18,8 @@ from cryptography.x509.oid import NameOID
 from init import shell
 
 
-def generate_selfsigned():
+def generate_self_signed(cert_path, key_path, ip=None,
+                         fingerprint_config=None):
     """Generate a self-signed certificate with associated keys.
 
     The certificate will have a fake CNAME and subjAltName since
@@ -28,26 +32,29 @@ def generate_selfsigned():
     via a secure channel.
     https://owasp.org/www-community/controls/Certificate_and_Public_Key_Pinning
     """
-    cert_path, key_path = (
-        Path(shell.config_get('config.cluster.tls-cert-path')),
-        Path(shell.config_get('config.cluster.tls-key-path')),
-    )
     # Do not generate a new certificate and key if there is already an existing
     # pair. TODO: improve this check and allow renewal.
     if cert_path.exists() and key_path.exists():
         return
 
-    dummy_cn = 'microstack.run'
     key = rsa.generate_private_key(
         public_exponent=65537,
         key_size=2048,
         backend=default_backend(),
     )
+    cn = socket.gethostname()
     common_name = x509.Name([
-        x509.NameAttribute(NameOID.COMMON_NAME, dummy_cn)
+        x509.NameAttribute(NameOID.COMMON_NAME, cn)
     ])
-    san = x509.SubjectAlternativeName([x509.DNSName(dummy_cn)])
+    if ip:
+        san = x509.SubjectAlternativeName(
+            [x509.DNSName(cn), x509.IPAddress(ipaddress.ip_address(ip))]
+        )
+    else:
+        san = x509.SubjectAlternativeName([x509.DNSName(cn)])
+
     basic_contraints = x509.BasicConstraints(ca=True, path_length=0)
+
     now = datetime.utcnow()
     cert = (
         x509.CertificateBuilder()
@@ -63,7 +70,8 @@ def generate_selfsigned():
     )
 
     cert_fprint = cert.fingerprint(hashes.SHA256()).hex()
-    shell.config_set(**{'config.cluster.fingerprint': cert_fprint})
+    if fingerprint_config:
+        shell.config_set(**{fingerprint_config: cert_fprint})
 
     serialized_cert = cert.public_bytes(encoding=serialization.Encoding.PEM)
     serialized_key = key.private_bytes(
@@ -73,3 +81,5 @@ def generate_selfsigned():
     )
     cert_path.write_bytes(serialized_cert)
     key_path.write_bytes(serialized_key)
+    check('chmod', '644', str(cert_path))
+    check('chmod', '600', str(key_path))

tools/launch/launch/main.py

@@ -207,7 +207,7 @@ Access it with `ssh -i {key_path} {username}@{ip}`\
     gate = check_output('snapctl', 'get', 'config.network.ext-gateway')
     port = check_output('snapctl', 'get', 'config.network.ports.dashboard')
 
-    print('You can also visit the OpenStack dashboard at http://{}:{}'.format(
+    print('You can also visit the OpenStack dashboard at https://{}:{}'.format(
           gate, port))