From 42420830f65c0a76ba8e7e1b6a8d2974dabb9cfa Mon Sep 17 00:00:00 2001 From: Jeffrey Zhang Date: Sun, 27 Mar 2016 00:39:07 +0800 Subject: [PATCH] Implement nova-ssh container Add a nova-ssh container to handle the `nova migrate` and `nova resize` case, in which the nova will use ssh to copy files between machines. Change-Id: Ie6675943f3aeabfbba8589d308d55b9c89d732db Closes-Bug: #1562141 --- ansible/roles/nova/defaults/main.yml | 6 ++++ ansible/roles/nova/tasks/config.yml | 12 ++++++++ ansible/roles/nova/tasks/do_reconfigure.yml | 15 ++++++---- ansible/roles/nova/tasks/pull.yml | 7 +++++ ansible/roles/nova/tasks/start_compute.yml | 15 ++++++++++ ansible/roles/nova/templates/id_rsa | 1 + ansible/roles/nova/templates/id_rsa.pub | 1 + ansible/roles/nova/templates/nova-ssh.json.j2 | 29 +++++++++++++++++++ ansible/roles/nova/templates/ssh_config.j2 | 4 +++ ansible/roles/nova/templates/sshd_config.j2 | 5 ++++ docker/nova/nova-base/Dockerfile.j2 | 6 ++-- docker/nova/nova-ssh/Dockerfile.j2 | 23 +++++++++++++++ docker/nova/nova-ssh/extend_start.sh | 20 +++++++++++++ etc/kolla/passwords.yml | 7 +++++ kolla/cmd/genpwd.py | 23 +++++++++++++++ requirements.txt | 1 + 16 files changed, 167 insertions(+), 8 deletions(-) create mode 100644 ansible/roles/nova/templates/id_rsa create mode 100644 ansible/roles/nova/templates/id_rsa.pub create mode 100644 ansible/roles/nova/templates/nova-ssh.json.j2 create mode 100644 ansible/roles/nova/templates/ssh_config.j2 create mode 100644 ansible/roles/nova/templates/sshd_config.j2 create mode 100644 docker/nova/nova-ssh/Dockerfile.j2 create mode 100644 docker/nova/nova-ssh/extend_start.sh diff --git a/ansible/roles/nova/defaults/main.yml b/ansible/roles/nova/defaults/main.yml index bb81cdeac2..7c5a54f151 100644 --- a/ansible/roles/nova/defaults/main.yml +++ b/ansible/roles/nova/defaults/main.yml @@ -32,6 +32,10 @@ nova_libvirt_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ do nova_libvirt_tag: "{{ openstack_release }}" nova_libvirt_image_full: "{{ nova_libvirt_image }}:{{ nova_libvirt_tag }}" +nova_ssh_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-nova-ssh" +nova_ssh_tag: "{{ openstack_release }}" +nova_ssh_image_full: "{{ nova_ssh_image }}:{{ nova_ssh_tag }}" + nova_conductor_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-nova-conductor" nova_conductor_tag: "{{ openstack_release }}" nova_conductor_image_full: "{{ nova_conductor_image }}:{{ nova_conductor_tag }}" @@ -74,3 +78,5 @@ nova_public_endpoint: "{{ public_protocol }}://{{ kolla_external_fqdn }}:{{ nova nova_logging_debug: "{{ openstack_logging_debug }}" openstack_nova_auth: "{'auth_url':'{{ openstack_auth.auth_url }}','username':'{{ openstack_auth.username }}','password':'{{ openstack_auth.password }}','project_name':'{{ openstack_auth.project_name }}'}" + +nova_ssh_port: "8022" diff --git a/ansible/roles/nova/tasks/config.yml b/ansible/roles/nova/tasks/config.yml index bd83120c22..8e2e7a8756 100644 --- a/ansible/roles/nova/tasks/config.yml +++ b/ansible/roles/nova/tasks/config.yml @@ -25,6 +25,7 @@ - "nova-novncproxy" - "nova-scheduler" - "nova-spicehtml5proxy" + - "nova-ssh" - name: Copying over config.json files for services template: @@ -40,6 +41,7 @@ - "nova-novncproxy" - "nova-scheduler" - "nova-spicehtml5proxy" + - "nova-ssh" - name: Copying over nova.conf merge_configs: @@ -68,3 +70,13 @@ template: src: "libvirtd.conf.j2" dest: "{{ node_config_directory }}/nova-libvirt/libvirtd.conf" + +- name: Copying files for nova-ssh + template: + src: "{{ item.src }}" + dest: "{{ node_config_directory }}/nova-ssh/{{ item.dest }}" + with_items: + - { src: "sshd_config.j2", dest: "sshd_config" } + - { src: "id_rsa", dest: "id_rsa" } + - { src: "id_rsa.pub", dest: "id_rsa.pub" } + - { src: "ssh_config.j2", dest: "ssh_config" } diff --git a/ansible/roles/nova/tasks/do_reconfigure.yml b/ansible/roles/nova/tasks/do_reconfigure.yml index 8891a4dea1..8bbd369d70 100644 --- a/ansible/roles/nova/tasks/do_reconfigure.yml +++ b/ansible/roles/nova/tasks/do_reconfigure.yml @@ -1,5 +1,5 @@ --- -- name: Ensuring the nova libvirt, conductor, api, consoleauth and scheduler containers are up +- name: Ensuring the nova libvirt, ssh, conductor, api, consoleauth and scheduler containers are up kolla_docker: name: "{{ item.name }}" action: "get_container_state" @@ -8,6 +8,7 @@ when: inventory_hostname in groups[item.group] with_items: - { name: nova_libvirt, group: compute } + - { name: nova_ssh, group: compute } - { name: nova_conductor, group: nova-conductor } - { name: nova_api, group: nova-api } - { name: nova_consoleauth, group: nova-consoleauth } @@ -55,7 +56,7 @@ - include: config.yml -- name: Check the configs for nova libvirt, conductor, api, consoleauth and scheduler containers +- name: Check the configs for nova libvirt, ssh, conductor, api, consoleauth and scheduler containers command: docker exec {{ item.name }} /usr/local/bin/kolla_set_configs --check changed_when: false failed_when: false @@ -63,6 +64,7 @@ when: inventory_hostname in groups[item.group] with_items: - { name: nova_libvirt, group: compute } + - { name: nova_ssh, group: compute } - { name: nova_conductor, group: nova-conductor } - { name: nova_api, group: nova-api } - { name: nova_consoleauth, group: nova-consoleauth } @@ -107,7 +109,7 @@ # NOTE(jeffrey4l): when config_strategy == 'COPY_ALWAYS' # and container env['KOLLA_CONFIG_STRATEGY'] == 'COPY_ONCE', # just remove the container and start again -- name: Containers config strategy for nova libvirt, conductor, api, consoleauth and scheduler containers +- name: Containers config strategy for nova libvirt, ssh, conductor, api, consoleauth and scheduler containers kolla_docker: name: "{{ item.name }}" action: "get_container_env" @@ -115,6 +117,7 @@ when: inventory_hostname in groups[item.group] with_items: - { name: nova_libvirt, group: compute } + - { name: nova_ssh, group: compute } - { name: nova_conductor, group: nova-conductor } - { name: nova_api, group: nova-api } - { name: nova_consoleauth, group: nova-consoleauth } @@ -156,7 +159,7 @@ - nova_console == 'spice' - inventory_hostname in groups['nova-spicehtml5proxy'] -- name: Remove the nova libvirt, conductor, api, consoleauth and scheduler containers +- name: Remove the nova libvirt, ssh, conductor, api, consoleauth and scheduler containers kolla_docker: name: "{{ item[0]['name'] }}" action: "remove_container" @@ -167,6 +170,7 @@ - item[2]['rc'] == 1 with_together: - [{ name: nova_libvirt, group: compute }, + { name: nova_ssh, group: compute }, { name: nova_conductor, group: nova-conductor }, { name: nova_api, group: nova-api }, { name: nova_consoleauth, group: nova-consoleauth }, @@ -246,7 +250,7 @@ - nova_console == 'spice' - remove_nova_spicehtml5proxy_container.changed -- name: Restart the nova libvirt, conductor, api, consoleauth and scheduler containers +- name: Restart the nova libvirt, ssh, conductor, api, consoleauth and scheduler containers kolla_docker: name: "{{ item[0]['name'] }}" action: "restart_container" @@ -257,6 +261,7 @@ - item[2]['rc'] == 1 with_together: - [{ name: nova_libvirt, group: compute }, + { name: nova_ssh, group: compute }, { name: nova_conductor, group: nova-conductor }, { name: nova_api, group: nova-api }, { name: nova_consoleauth, group: nova-consoleauth }, diff --git a/ansible/roles/nova/tasks/pull.yml b/ansible/roles/nova/tasks/pull.yml index a8d0a9f52a..b761f0c451 100644 --- a/ansible/roles/nova/tasks/pull.yml +++ b/ansible/roles/nova/tasks/pull.yml @@ -45,6 +45,13 @@ image: "{{ nova_libvirt_image_full }}" when: inventory_hostname in groups['compute'] +- name: Pulling nova-ssh image + kolla_docker: + action: "pull_image" + common_options: "{{ docker_common_options }}" + image: "{{ nova_ssh_image_full }}" + when: inventory_hostname in groups['compute'] + - name: Pulling nova-novncproxy image kolla_docker: action: "pull_image" diff --git a/ansible/roles/nova/tasks/start_compute.yml b/ansible/roles/nova/tasks/start_compute.yml index 1540f4673f..f14b2b0c49 100644 --- a/ansible/roles/nova/tasks/start_compute.yml +++ b/ansible/roles/nova/tasks/start_compute.yml @@ -64,3 +64,18 @@ when: - inventory_hostname in groups['compute'] - enable_nova_fake | bool + +- name: Staring nova-ssh container + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + image: "{{ nova_ssh_image_full }}" + name: "nova_ssh" + volumes: + - "{{ node_config_directory }}/nova-ssh/:{{ container_config_directory }}/:ro" + - "kolla_logs:/var/log/kolla" + - "nova_compute:/var/lib/nova" + - "heka_socket:/var/lib/kolla/heka/" + # TODO(jeffrey4l): how to handle the nova-compute-fake and + # nova-compute-ironic + when: inventory_hostname in groups['compute'] diff --git a/ansible/roles/nova/templates/id_rsa b/ansible/roles/nova/templates/id_rsa new file mode 100644 index 0000000000..173a4b3e12 --- /dev/null +++ b/ansible/roles/nova/templates/id_rsa @@ -0,0 +1 @@ +{{ nova_ssh_key.private_key }} diff --git a/ansible/roles/nova/templates/id_rsa.pub b/ansible/roles/nova/templates/id_rsa.pub new file mode 100644 index 0000000000..16bd674f22 --- /dev/null +++ b/ansible/roles/nova/templates/id_rsa.pub @@ -0,0 +1 @@ +{{ nova_ssh_key.public_key }} diff --git a/ansible/roles/nova/templates/nova-ssh.json.j2 b/ansible/roles/nova/templates/nova-ssh.json.j2 new file mode 100644 index 0000000000..1fb041ecc9 --- /dev/null +++ b/ansible/roles/nova/templates/nova-ssh.json.j2 @@ -0,0 +1,29 @@ +{ + "command": "/usr/sbin/sshd -D", + "config_files": [ + { + "source": "{{ container_config_directory }}/sshd_config", + "dest": "/etc/ssh/sshd_config", + "owner": "root", + "perm": "0644" + }, + { + "source": "{{ container_config_directory }}/ssh_config", + "dest": "/var/lib/nova/.ssh/config", + "owner": "nova", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/id_rsa", + "dest": "/var/lib/nova/.ssh/id_rsa", + "owner": "nova", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/id_rsa.pub", + "dest": "/var/lib/nova/.ssh/authorized_keys", + "owner": "nova", + "perm": "0600" + } + ] +} diff --git a/ansible/roles/nova/templates/ssh_config.j2 b/ansible/roles/nova/templates/ssh_config.j2 new file mode 100644 index 0000000000..7c5c962f9d --- /dev/null +++ b/ansible/roles/nova/templates/ssh_config.j2 @@ -0,0 +1,4 @@ +Host * + StrictHostKeyChecking no + UserKnownHostsFile /dev/null + port {{ nova_ssh_port }} diff --git a/ansible/roles/nova/templates/sshd_config.j2 b/ansible/roles/nova/templates/sshd_config.j2 new file mode 100644 index 0000000000..ba6c8df8cc --- /dev/null +++ b/ansible/roles/nova/templates/sshd_config.j2 @@ -0,0 +1,5 @@ +Port {{ nova_ssh_port }} +ListenAddress {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }} + +SyslogFacility AUTHPRIV +UsePAM yes diff --git a/docker/nova/nova-base/Dockerfile.j2 b/docker/nova/nova-base/Dockerfile.j2 index f9f465f16a..e34f3f228d 100644 --- a/docker/nova/nova-base/Dockerfile.j2 +++ b/docker/nova/nova-base/Dockerfile.j2 @@ -45,11 +45,11 @@ RUN apt-get install -y --no-install-recommends \ ADD nova-base-archive /nova-base-source RUN ln -s nova-base-source/* nova \ - && useradd --user-group nova \ + && useradd --user-group --home-dir /var/lib/nova nova \ && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt /nova \ - && mkdir -p /etc/nova /home/nova /var/lib/nova \ + && mkdir -p /etc/nova /var/lib/nova \ && cp -r /nova/etc/nova/* /etc/nova/ \ - && chown -R nova: /etc/nova /home/nova /var/lib/nova \ + && chown -R nova: /etc/nova /var/lib/nova \ && sed -i 's|^exec_dirs.*|exec_dirs=/var/lib/kolla/venv/bin,/sbin,/usr/sbin,/bin,/usr/bin,/usr/local/bin,/usr/local/sbin|g' /etc/nova/rootwrap.conf COPY nova_sudoers /etc/sudoers.d/nova_sudoers diff --git a/docker/nova/nova-ssh/Dockerfile.j2 b/docker/nova/nova-ssh/Dockerfile.j2 new file mode 100644 index 0000000000..7bc859ef1b --- /dev/null +++ b/docker/nova/nova-ssh/Dockerfile.j2 @@ -0,0 +1,23 @@ +FROM {{ namespace }}/{{ image_prefix }}nova-base:{{ tag }} +MAINTAINER {{ maintainer }} + +{% if base_distro in ['centos', 'fedora', 'oraclelinux', 'rhel'] %} + +RUN yum -y install \ + openssh-server \ + && yum clean all + +{% elif base_distro in ['ubuntu', 'debian'] %} + +RUN apt-get install -y --no-install-recommends \ + openssh-server \ + && apt-get clean \ + && mkdir -p /var/run/sshd \ + && chmod 0755 /var/run/sshd + +{% endif %} + +COPY extend_start.sh /usr/local/bin/kolla_extend_start +RUN chmod 755 /usr/local/bin/kolla_extend_start + +{{ include_footer }} diff --git a/docker/nova/nova-ssh/extend_start.sh b/docker/nova/nova-ssh/extend_start.sh new file mode 100644 index 0000000000..da05e4379a --- /dev/null +++ b/docker/nova/nova-ssh/extend_start.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +if [[ ! -L /dev/log ]]; then + ln -sf /var/lib/kolla/heka/log /dev/log +fi + +SSH_HOST_KEY_TYPES=( "rsa" "dsa" "ecdsa" "ed25519" ) + +for key_type in ${SSH_HOST_KEY_TYPES[@]}; do + KEY_PATH=/etc/ssh/ssh_host_${key_type}_key + if [[ ! -f "${KEY_PATH}" ]]; then + ssh-keygen -q -t ${key_type} -f ${KEY_PATH} -N "" + fi +done + +mkdir -p /var/lib/nova/.ssh + +if [[ $(stat -c %U:%G /var/lib/nova/.ssh) != "nova:nova" ]]; then + chown nova: /var/lib/nova/.ssh +fi diff --git a/etc/kolla/passwords.yml b/etc/kolla/passwords.yml index 12b2d6bf5e..de41e700c4 100644 --- a/etc/kolla/passwords.yml +++ b/etc/kolla/passwords.yml @@ -65,6 +65,13 @@ manila_keystone_password: memcache_secret_key: +nova_ssh_private_key: +nova_ssh_public_key: + +nova_ssh_key: + private_key: + public_key: + #################### # RabbitMQ options #################### diff --git a/kolla/cmd/genpwd.py b/kolla/cmd/genpwd.py index 728dd458b9..70dbc0357c 100755 --- a/kolla/cmd/genpwd.py +++ b/kolla/cmd/genpwd.py @@ -12,16 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import random import string import uuid import yaml +from Crypto.PublicKey import RSA + + +def generate_RSA(bits=2048): + new_key = RSA.generate(bits, os.urandom) + private_key = new_key.exportKey("PEM") + public_key = new_key.publickey().exportKey("OpenSSH") + return private_key, public_key + def main(): # These keys should be random uuids uuid_keys = ['ceph_cluster_fsid', 'rbd_secret_uuid'] + # SSH key pair + ssh_keys = ['nova_ssh_key'] + # If these keys are None, leave them as None blank_keys = ['docker_registry_password'] @@ -32,6 +45,16 @@ def main(): passwords = yaml.load(f.read()) for k, v in passwords.items(): + if (k in ssh_keys and + (v is None + or v.get('public_key') is None + and v.get('private_key') is None)): + private_key, public_key = generate_RSA() + passwords[k] = { + 'private_key': private_key, + 'public_key': public_key + } + continue if v is None: if k in blank_keys and v is None: continue diff --git a/requirements.txt b/requirements.txt index 7995bf1523..9646d20463 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ oslo.config>=3.7.0 # Apache-2.0 graphviz>=0.4.0 # MIT License beautifulsoup4 # MIT setuptools>=16.0 # PSF/ZPL +pycrypto>=2.6 # Public Domain