Support storing passwords in Hashicorp Vault

This commit adds two new cli commands to allow an operator
to read and write passwords into a configured Hashicorp Vault
KV.

Change-Id: Icf0eaf7544fcbdf7b83f697cc711446f47118a4d

doc/source/user/operating-kolla.rst

@@ -235,6 +235,33 @@ For example:
    To alter this behavior, and remove such entries, use the ``--clean``
    argument when invoking ``kolla-mergepwd``.
 
+Hashicorp Vault can be used as an alternative to Ansible Vault for storing
+passwords generated by Kolla Ansible. To use Hashicorp Vault as the secrets
+store you will first need to generate the passwords, and then you can
+save them into an existing KV using the following command:
+
+.. code-block:: console
+
+   kolla-writepwd \
+   --passwords /etc/kolla/passwords.yml \
+   --vault-addr <VAULT_ADDRESS> \
+   --vault-token <VAULT_TOKEN>
+
+.. note::
+
+   For a full list of ``kolla-writepwd`` arguments, use the ``--help``
+   argument when invoking ``kolla-writepwd``.
+
+To read passwords from Hashicorp Vault and generate a passwords.yml:
+
+.. code-block:: console
+
+   mv kolla-ansible/etc/kolla/passwords.yml /etc/kolla/passwords.yml
+   kolla-readpwd \
+   --passwords /etc/kolla/passwords.yml \
+   --vault-addr <VAULT_ADDRESS> \
+   --vault-token <VAULT_TOKEN>
+
 Tools
 -----
 

kolla_ansible/cmd/readpwd.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import sys
+
+import hvac
+import yaml
+
+from kolla_ansible.hashi_vault import hashicorp_vault_client
+
+
+def readpwd(passwords_file, vault_kv_path, vault_mount_point, vault_namespace,
+            vault_addr, vault_role_id, vault_secret_id, vault_token,
+            vault_cacert):
+
+    with open(passwords_file, 'r') as f:
+        passwords = yaml.safe_load(f.read())
+
+    if not isinstance(passwords, dict):
+        print("ERROR: Passwords file not in expected key/value format")
+        sys.exit(1)
+
+    client = hashicorp_vault_client(vault_namespace, vault_addr, vault_role_id,
+                                    vault_secret_id, vault_token, vault_cacert)
+
+    vault_kv_passwords = dict()
+    for password_key in passwords:
+        try:
+            password_data = client.secrets.kv.v2.read_secret_version(
+                mount_point=vault_mount_point,
+                path="{}/{}".format(vault_kv_path, password_key))
+        except hvac.exceptions.InvalidPath:
+            # Ignore passwords that are not found in Vault
+            print("WARNING: '%s' not found in Vault" % password_key)
+            vault_kv_passwords[password_key] = None
+            continue
+        try:
+            vault_kv_passwords[password_key] =\
+                password_data['data']['data']['password']
+        except KeyError:
+            vault_kv_passwords[password_key] = password_data['data']['data']
+
+    with open(passwords_file, 'w') as f:
+        yaml.safe_dump(vault_kv_passwords, f)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-p', '--passwords', type=str,
+        default=os.path.abspath('/etc/kolla/passwords.yml'),
+        help='Path to the passwords.yml file')
+    parser.add_argument(
+        '-kv', '--vault-mount-point', type=str,
+        default='kv',
+        help='Path to the KV mount point')
+    parser.add_argument(
+        '-kvp', '--vault-kv-path', type=str,
+        default='kolla_passwords',
+        help='Path to store passwords within your configured KV mount point')
+    parser.add_argument(
+        '-n', '--vault-namespace', type=str,
+        default='',
+        help='Vault namespace (enterprise only)')
+    parser.add_argument(
+        '-v', '--vault-addr', type=str,
+        required=True,
+        help='Address to connect to an existing Hashicorp Vault')
+    parser.add_argument(
+        '-r', '--vault-role-id', type=str,
+        default='',
+        help='Role-ID to authenticate to Vault. This must be used in '
+             'conjunction with --secret-id')
+    parser.add_argument(
+        '-s', '--vault-secret-id', type=str,
+        default='',
+        help='Secret-ID to authenticate to Vault. This must be used in '
+             'conjunction with --role-id')
+    parser.add_argument(
+        '-t', '--vault-token', type=str,
+        default='',
+        help='Vault token to authenticate to Vault')
+    parser.add_argument(
+        '-c', '--vault-cacert', type=str,
+        default='',
+        help='Path to CA certificate file')
+
+    args = parser.parse_args()
+    passwords_file = os.path.expanduser(args.passwords)
+    vault_kv_path = args.vault_kv_path
+    vault_mount_point = args.vault_mount_point
+    vault_namespace = args.vault_namespace
+    vault_addr = args.vault_addr
+    vault_role_id = args.vault_role_id
+    vault_secret_id = args.vault_secret_id
+    vault_token = args.vault_token
+    vault_cacert = os.path.expanduser(args.vault_cacert)
+
+    readpwd(passwords_file, vault_kv_path, vault_mount_point, vault_namespace,
+            vault_addr, vault_role_id, vault_secret_id, vault_token,
+            vault_cacert)
+
+
+if __name__ == '__main__':
+    main()

kolla_ansible/cmd/writepwd.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import sys
+
+import hvac
+import yaml
+
+from kolla_ansible.hashi_vault import hashicorp_vault_client
+
+
+def writepwd(passwords_file, vault_kv_path, vault_mount_point, vault_namespace,
+             vault_addr, vault_role_id, vault_secret_id, vault_token,
+             vault_cacert):
+    with open(passwords_file, 'r') as f:
+        passwords = yaml.safe_load(f.read())
+
+    if not isinstance(passwords, dict):
+        print("ERROR: Passwords file not in expected key/value format")
+        sys.exit(1)
+
+    client = hashicorp_vault_client(vault_namespace, vault_addr, vault_role_id,
+                                    vault_secret_id, vault_token, vault_cacert)
+
+    for key, value in passwords.items():
+        # Ignore empty values
+        if not value:
+            continue
+
+        if isinstance(value, str):
+            value = dict(password=value)
+
+        try:
+            remote_value = client.secrets.kv.v2.read_secret_version(
+                mount_point=vault_mount_point,
+                path="{}/{}".format(vault_kv_path, key))
+        except hvac.exceptions.InvalidPath:
+            # Add to KV if value does not exists
+            remote_value = None
+
+        # Update KV is value has changed or it does not exist
+        if not remote_value or remote_value['data']['data'] != value:
+            client.secrets.kv.v2.create_or_update_secret(
+                mount_point=vault_mount_point,
+                path="{}/{}".format(vault_kv_path, key),
+                secret=value)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-p', '--passwords', type=str,
+        default=os.path.abspath('/etc/kolla/passwords.yml'),
+        help='Path to the passwords.yml file')
+    parser.add_argument(
+        '-kv', '--vault-mount-point', type=str,
+        default='kv',
+        help='Path to the KV mount point')
+    parser.add_argument(
+        '-kvp', '--vault-kv-path', type=str,
+        default='kolla_passwords',
+        help='Path to store passwords within your configured KV mount point')
+    parser.add_argument(
+        '-n', '--vault-namespace', type=str,
+        default='',
+        help='Vault namespace (enterprise only)')
+    parser.add_argument(
+        '-v', '--vault-addr', type=str,
+        required=True,
+        help='Address to connect to an existing Hashicorp Vault')
+    parser.add_argument(
+        '-r', '--vault-role-id', type=str,
+        default='',
+        help='Role-ID to authenticate to Vault. This must be used in '
+             'conjunction with --secret-id')
+    parser.add_argument(
+        '-s', '--vault-secret-id', type=str,
+        default='',
+        help='Secret-ID to authenticate to Vault. This must be used in '
+             'conjunction with --role-id')
+    parser.add_argument(
+        '-t', '--vault-token', type=str,
+        default='',
+        help='Vault token to authenticate to Vault')
+    parser.add_argument(
+        '-c', '--vault-cacert', type=str,
+        default='',
+        help='Path to CA certificate file')
+
+    args = parser.parse_args()
+    passwords_file = os.path.expanduser(args.passwords)
+    vault_kv_path = args.vault_kv_path
+    vault_mount_point = args.vault_mount_point
+    vault_namespace = args.vault_namespace
+    vault_addr = args.vault_addr
+    vault_role_id = args.vault_role_id
+    vault_secret_id = args.vault_secret_id
+    vault_token = args.vault_token
+    vault_cacert = os.path.expanduser(args.vault_cacert)
+
+    writepwd(passwords_file, vault_kv_path, vault_mount_point, vault_namespace,
+             vault_addr, vault_role_id, vault_secret_id, vault_token,
+             vault_cacert)
+
+
+if __name__ == '__main__':
+    main()

kolla_ansible/hashi_vault.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+import hvac
+
+
+def hashicorp_vault_client(vault_namespace, vault_addr, vault_role_id,
+                           vault_secret_id, vault_token, vault_cacert):
+    """Connect to a Vault sever and create a client.
+
+    :param vault_namespace: Vault namespace (enterprise only).
+    :param vault_addr: Address to connect to an existing Hashicorp Vault.
+    :param vault_role_id: Role-ID to authenticate to Vault. This must be used
+    in conjunction with --secret-id.
+    :param vault_secret_id: Secret-ID to authenticate to Vault. This must be
+    used in conjunction with --role-id.
+    :param vault_token: Vault token to authenticate to Vault.
+    :param vault_cacert: Path to CA certificate file.
+    :returns: Hashicorp Vault Client (hvac.Client).
+    """
+
+    if any([vault_role_id, vault_secret_id]):
+        if vault_token:
+            print("ERROR: Vault token cannot be used at the same time as "
+                  "role-id and secret-id")
+            sys.exit(1)
+        if not all([vault_role_id, vault_secret_id]):
+            print("ERROR: role-id and secret-id must be provided together")
+            sys.exit(1)
+    elif not vault_token:
+        print("ERROR: You must provide either a Vault token or role-id and "
+              "secret-id")
+        sys.exit(1)
+
+    # Authenticate to Hashicorp Vault
+    if vault_cacert != "":
+        os.environ['REQUESTS_CA_BUNDLE'] = vault_cacert
+
+    if vault_token != "":  # nosec
+        client = hvac.Client(url=vault_addr, token=vault_token,
+                             namespace=vault_namespace)
+    else:
+        client = hvac.Client(url=vault_addr, namespace=vault_namespace)
+        client.auth_approle(vault_role_id, vault_secret_id)
+
+    if not client.is_authenticated():
+        print('Failed to authenticate to vault')
+        sys.exit(1)
+
+    return client

releasenotes/notes/support-hashicorp-vault-for-kolla-passwords-76a5b4ece6a4df07.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Adds functionality to allow passwords that are generated for Kolla
+    Ansible to be stored in Hashicorp Vault. Use new CLI commands
+    `kolla-readpwd` and `kolla-writepwd` to read and write Kolla Ansible
+    passwords to a configured Hashicorp Vault kv secrets engine.

requirements.txt

@@ -15,3 +15,6 @@ Jinja2>=2.10 # BSD License (3 clause)
 
 # Ansible's json_query
 jmespath>=0.9.3 # MIT
+
+# Hashicorp Vault
+hvac>=0.10.1

setup.cfg

@@ -45,3 +45,5 @@ scripts =
 console_scripts =
     kolla-genpwd = kolla_ansible.cmd.genpwd:main
     kolla-mergepwd = kolla_ansible.cmd.mergepwd:main
+    kolla-writepwd = kolla_ansible.cmd.writepwd:main
+    kolla-readpwd = kolla_ansible.cmd.readpwd:main

tests/run-hashi-vault.yml

@@ -0,0 +1,75 @@
+---
+- hosts: all
+  any_errors_fatal: true
+  tasks:
+    # NOTE(yoctozepto): setting vars as facts for all to have them around in all the plays
+    - name: set facts for commonly used variables
+      set_fact:
+        kolla_ansible_src_dir: "{{ ansible_env.PWD }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible"
+        upper_constraints_file: "{{ ansible_env.HOME }}/src/opendev.org/openstack/requirements/upper-constraints.txt"
+        pip_user_path_env:
+          PATH: "{{ ansible_env.HOME + '/.local/bin:' + ansible_env.PATH }}"
+
+- hosts: primary
+  any_errors_fatal: true
+  environment: "{{ pip_user_path_env }}"
+  tasks:
+    - name: ensure /etc/kolla exists
+      file:
+        path: "/etc/kolla"
+        state: "directory"
+        mode: 0777
+      become: true
+
+    # NOTE(mgoddard): We need a recent pip to install the latest cryptography
+    # library. See https://github.com/pyca/cryptography/issues/5753
+    - name: install pip 19.1.1+
+      pip:
+        name: "pip>=19.1.1"
+        executable: "pip3"
+        extra_args: "--user"
+
+    - name: install kolla-ansible and dependencies
+      pip:
+        name:
+          - "{{ kolla_ansible_src_dir }}"
+        executable: "pip3"
+        extra_args: "-c {{ upper_constraints_file }} --user"
+
+    - name: copy passwords.yml file
+      copy:
+        src: "{{ kolla_ansible_src_dir }}/etc/kolla/passwords.yml"
+        dest: /etc/kolla/passwords.yml
+        remote_src: true
+
+    - name: generate passwords
+      command: kolla-genpwd
+
+    # At this point we have generated all necessary configuration, and are
+    # ready to test Hashicorp Vault.
+    - name: Run test-hashicorp-vault-passwords.sh script
+      script:
+        cmd: test-hashicorp-vault-passwords.sh
+        executable: /bin/bash
+        chdir: "{{ kolla_ansible_src_dir }}"
+      environment:
+        BASE_DISTRO: "{{ base_distro }}"
+
+    - name: Read template file
+      slurp:
+        src: "/etc/kolla/passwords.yml"
+      register: template_file
+
+    - name: Read generated file
+      slurp:
+        src: "/tmp/passwords-hashicorp-vault.yml"
+      register: generated_file
+
+    # This test will load in the original input file and the one that was
+    # generated by Vault and ensure that the keys are the same in both files.
+    # This ensures that we are not missing any passwords.
+    - name: Check passwords that were written to Vault are as expected
+      vars:
+        input_passwords: "{{ template_file['content'] | b64decode | from_yaml | sort }}"
+        output_passwords: "{{ generated_file['content'] | b64decode | from_yaml | sort }}"
+      assert: { that: "input_passwords == output_passwords" }

tests/test-hashicorp-vault-passwords.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+export PYTHONUNBUFFERED=1
+
+function install_vault {
+    if [[ "debian" == $BASE_DISTRO ]]; then
+        curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add -
+        sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+        sudo apt-get update -y && sudo apt-get install -y vault jq
+    else
+        sudo dnf install -y yum-utils
+        sudo dnf config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
+        sudo dnf install -y vault jq
+    fi
+}
+
+function start_vault {
+    nohup vault server --dev &
+    # Give Vault some time to warm up
+    sleep 10
+}
+
+function test_vault {
+    TOKEN=$(vault token create -address 'http://127.0.0.1:8200' -format json | jq '.auth.client_token' --raw-output)
+    echo "${TOKEN}" | vault login -address 'http://127.0.0.1:8200' -
+    vault kv put -address 'http://127.0.0.1:8200' secret/foo data=bar
+}
+
+function test_writepwd {
+    TOKEN=$(vault token create -address 'http://127.0.0.1:8200' -format json | jq '.auth.client_token' --raw-output)
+    kolla-writepwd \
+        --passwords /etc/kolla/passwords.yml \
+        --vault-addr 'http://127.0.0.1:8200' \
+        --vault-token ${TOKEN} \
+        --vault-mount-point secret
+}
+
+function test_readpwd {
+    TOKEN=$(vault token create -address 'http://127.0.0.1:8200' -format json | jq '.auth.client_token' --raw-output)
+    cp etc/kolla/passwords.yml /tmp/passwords-hashicorp-vault.yml
+    kolla-readpwd \
+        --passwords /tmp/passwords-hashicorp-vault.yml \
+        --vault-addr 'http://127.0.0.1:8200' \
+        --vault-token ${TOKEN} \
+        --vault-mount-point secret
+}
+
+function teardown {
+    pkill vault
+}
+
+function test_hashicorp_vault_passwords {
+    echo "Setting up development Vault server..."
+    install_vault
+    start_vault
+    test_vault
+    echo "Write passwords to Hashicorp Vault..."
+    test_writepwd
+    echo "Read passwords from Hashicorp Vault..."
+    test_readpwd
+    echo "Cleaning up..."
+    teardown
+}
+
+test_hashicorp_vault_passwords

tools/read_passwords.py

@@ -0,0 +1 @@
+../kolla_ansible/cmd/readpwd.py

tools/write_passwords.py

@@ -0,0 +1 @@
+../kolla_ansible/cmd/writepwd.py

zuul.d/base.yaml

@@ -230,3 +230,25 @@
       - ^tests/test-prometheus-efk.sh
     vars:
       scenario: prometheus-efk
+
+- job:
+    name: kolla-ansible-hashi-vault-base
+    run: tests/run-hashi-vault.yml
+    required-projects:
+      - openstack/kolla-ansible
+      - openstack/requirements
+    voting: false
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*
+      - ^releasenotes/.*$
+      - ^deploy-guide/.*$
+      - ^test-requirements.txt$
+      - ^etc/kolla/globals.yml$
+      - ^tox.ini$
+      - ^\..+
+      - ^LICENSE$
+      - ^contrib/
+      - ^specs/
+      - ^kolla_ansible/tests/
+      - ^zuul\.d/

zuul.d/jobs.yaml

@@ -378,3 +378,10 @@
     vars:
       base_distro: ubuntu
       install_type: source
+
+- job:
+    name: kolla-ansible-centos8s-hashi-vault
+    parent: kolla-ansible-hashi-vault-base
+    nodeset: kolla-ansible-centos8s
+    vars:
+      base_distro: centos

zuul.d/project.yaml

@@ -52,6 +52,7 @@
         - kolla-ansible-ubuntu-source-cephadm
         - kolla-ansible-centos8s-source-upgrade-cephadm
         - kolla-ansible-ubuntu-source-upgrade-cephadm
+        - kolla-ansible-centos8s-hashi-vault
     check-arm64:
       jobs:
         - kolla-ansible-debian-source-aarch64