diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 3a4419723b..e4e903f9f1 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -140,6 +140,7 @@ enable_nova: "yes" enable_rabbitmq: "yes" # Additional optional OpenStack services are specified here +enable_ceph: "no" enable_cinder: "no" enable_heat: "yes" enable_horizon: "yes" diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index 8b782f72ac..d820c3e606 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -54,6 +54,12 @@ control [murano:children] control +[ceph-mon:children] +control + +[ceph-osd:children] +storage + # Additional control implemented here. These groups allow you to control which # services run on which hosts at a per-service level. diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index 924c4ac844..51f9d055ca 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -62,6 +62,12 @@ control [murano:children] control +[ceph-mon:children] +control + +[ceph-osd:children] +storage + # Additional control implemented here. These groups allow you to control which # services run on which hosts at a per-service level. diff --git a/ansible/library/bslurp.py b/ansible/library/bslurp.py new file mode 100644 index 0000000000..485b4c3b46 --- /dev/null +++ b/ansible/library/bslurp.py @@ -0,0 +1,189 @@ +#!/usr/bin/python + +# Copyright 2015 Sam Yaple +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This module has been relicensed from the source below: +# https://github.com/SamYaple/yaodu/blob/master/ansible/library/bslurp + +DOCUMENTATION = ''' +--- +module: bslurp +short_description: Slurps a file from a remote node +description: + - Used for fetching a binary blob containing the file, then push that file + to other hosts. +options: + src: + description: + - File to fetch. When dest is used, src is expected to be a str with data + required: True + type: str + compress: + description: + - Compress file with zlib + default: True + type: bool + dest: + description: + - Where to write out binary blob + required: False + type: str + mode: + description: + - Destination file permissions + default: '0644' + type: str + sha1: + description: + - sha1 hash of the underlying data + default: None + type: bool +author: Sam Yaple +''' + +EXAMPLES = ''' +Distribute a file from single to many host: + +- hosts: web_servers + tasks: + - name: Pull in web config + bslurp: src="/path/to/file" + register: file_data + run_once: True + - name: Push if changed + bslurp: + src: "{{ file_data.content }}" + dest: "{{ file_data.source }}" + mode: "{{ file_data.mode }}" + sha1: "{{ file_data.sha1 }}" + +Distribute multiple files from single to many host: + +- hosts: web_servers + tasks: + - name: Pull in web config + bslurp: src="{{ item }}" + with_items: + - "/path/to/file1" + - "/path/to/file2" + - "/path/to/file3" + register: file_data + run_once: True + - name: Push if changed + bslurp: + src: "{{ item.content }}" + dest: "{{ item.source }}" + mode: "{{ item.mode }}" + sha1: "{{ item.sha1 }}" + with_items: file_data.results + +Distribute a to file many host without compression; Change permissions on dest: + +- hosts: web_servers + tasks: + - name: Pull in web config + bslurp: src="/path/to/file" + register: file_data + run_once: True + - name: Push if changed + bslurp: + src: "{{ file_data.content }}" + dest: "/new/path/to/file" + mode: "0777" + compress: False + sha1: "{{ file_data.sha1 }}" +''' + +import base64 +import hashlib +import os +import sys +import zlib + +def copy_from_host(module): + compress = module.params.get('compress') + src = module.params.get('src') + + if not os.path.exists(src): + module.fail_json(msg="file not found: {}".format(src)) + if not os.access(src, os.R_OK): + module.fail_json(msg="file is not readable: {}".format(src)) + + mode = oct(os.stat(src).st_mode & 0777) + + with open(src, 'rb') as f: + raw_data = f.read() + + sha1 = hashlib.sha1(raw_data).hexdigest() + data = zlib.compress(raw_data) if compress else raw_data + + module.exit_json(content=base64.b64encode(data), sha1=sha1, mode=mode, + source=src) + + +def copy_to_host(module): + compress = module.params.get('compress') + dest = module.params.get('dest') + mode = int(module.params.get('mode'), 0) + sha1 = module.params.get('sha1') + src = module.params.get('src') + + data = base64.b64decode(src) + raw_data = zlib.decompress(data) if compress else data + + if sha1: + if os.path.exists(dest): + if os.access(dest, os.R_OK): + with open(dest, 'rb') as f: + if hashlib.sha1(f.read()).hexdigest() == sha1: + module.exit_json(changed=False) + else: + module.exit_json(failed=True, changed=False, + msg='file is not accessible: {}'.format(dest)) + + if sha1 != hashlib.sha1(raw_data).hexdigest(): + module.exit_json(failed=True, changed=False, + msg='sha1 sum does not match data') + + with os.fdopen(os.open(dest, os.O_WRONLY | os.O_CREAT, mode), 'wb') as f: + f.write(raw_data) + + module.exit_json(changed=True) + +def main(): + module = AnsibleModule( + argument_spec = dict( + compress = dict(default=True, type='bool'), + dest = dict(type='str'), + mode = dict(default='0644', type='str'), + sha1 = dict(default=None, type='str'), + src = dict(required=True, type='str') + ) + ) + + dest = module.params.get('dest') + + try: + if dest: + copy_to_host(module) + else: + copy_from_host(module) + except Exception as e: + module.exit_json(failed=True, changed=True, msg=repr(e)) + +# import module snippets +from ansible.module_utils.basic import * +if __name__ == '__main__': + main() diff --git a/ansible/library/find_disks.py b/ansible/library/find_disks.py new file mode 100644 index 0000000000..fe2a638b25 --- /dev/null +++ b/ansible/library/find_disks.py @@ -0,0 +1,86 @@ +#!/usr/bin/python + +# Copyright 2015 Sam Yaple +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This module has been relicensed from the source below: +# https://github.com/SamYaple/yaodu/blob/master/ansible/library/ceph_osd_list + +DOCUMENTATION = ''' +--- +module: find_disks +short_description: Return list of devices containing a specfied label +description: + - This will return a list of all devices with a GPT partition label with + the name specified. +options: + partition_name: + description: + - Partition name + required: True + type: bool +author: Sam Yaple +''' + +EXAMPLES = ''' +- hosts: ceph-osd + tasks: + - name: Return all valid formated devices with the name KOLLA_CEPH_OSD + ceph_osd_list: + partition_name: 'KOLLA_CEPH_OSD' + register: osds +''' + +import sys +import subprocess + + +def main(): + module = AnsibleModule( + argument_spec = dict( + partition_name = dict(required=True, type='str') + ) + ) + + partition_name = module.params.get('partition_name') + + try: + # This should all really be done differently. Unfortunately there is no + # decent python library for dealing with disks like we need to here. + disks = subprocess.check_output("parted -l", shell=True).split('\n') + ret = list() + + for line in disks: + d = line.split(' ') + if d[0] == 'Disk': + dev = d[1][:-1] + + if line.find(partition_name) != -1: + # This process returns an error code when no results return + # We can ignore that, it is safe + p = subprocess.Popen("blkid " + dev + "*", shell=True, stdout=subprocess.PIPE) + fs_uuid = p.communicate()[0] + # The dev doesn't need to have a uuid, will be '' otherwise + if fs_uuid: + fs_uuid = fs_uuid.split('"')[1] + ret.append({'device': dev, 'fs_uuid': fs_uuid}) + + module.exit_json(disks=ret) + except Exception as e: + module.exit_json(failed=True, msg=repr(e)) + +# import module snippets +from ansible.module_utils.basic import * +if __name__ == '__main__': + main() diff --git a/ansible/roles/ceph/defaults/main.yml b/ansible/roles/ceph/defaults/main.yml new file mode 100644 index 0000000000..08fffde130 --- /dev/null +++ b/ansible/roles/ceph/defaults/main.yml @@ -0,0 +1,24 @@ +--- +project_name: "ceph" + + +#################### +# Docker +#################### +ceph_mon_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-ceph-mon" +ceph_mon_tag: "{{ openstack_release }}" +ceph_mon_image_full: "{{ ceph_mon_image }}:{{ ceph_mon_tag }}" + +ceph_osd_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-ceph-osd" +ceph_osd_tag: "{{ openstack_release }}" +ceph_osd_image_full: "{{ ceph_osd_image }}:{{ ceph_osd_tag }}" + +ceph_data_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-data" +ceph_data_tag: "{{ openstack_release }}" +ceph_data_image_full: "{{ ceph_data_image }}:{{ ceph_data_tag }}" + + +#################### +# Ceph +#################### +osd_initial_weight: "1" diff --git a/ansible/roles/ceph/tasks/bootstrap_mons.yml b/ansible/roles/ceph/tasks/bootstrap_mons.yml new file mode 100644 index 0000000000..853056b885 --- /dev/null +++ b/ansible/roles/ceph/tasks/bootstrap_mons.yml @@ -0,0 +1,57 @@ +--- +- name: Cleaning up temp file on localhost + local_action: file path=/tmp/kolla_ceph_cluster state=absent + changed_when: False + always_run: True + run_once: True + +- name: Creating temp file on localhost + local_action: copy content=None dest=/tmp/kolla_ceph_cluster mode=0600 + changed_when: False + always_run: True + run_once: True + +# TODO(SamYaple): Improve failed_when check +- name: Checking if a previous cluster exists + command: docker exec ceph_mon_data stat /etc/ceph/ceph.monmap + register: exists + changed_when: False + failed_when: False + always_run: True + +- name: Writing hostname of host with existing cluster files to temp file + local_action: copy content={{ ansible_hostname }} dest=/tmp/kolla_ceph_cluster mode=0600 + changed_when: False + always_run: True + when: exists.rc == 0 + +- name: Registering host from temp file + set_fact: + delegate_host: "{{ lookup('file', '/tmp/kolla_ceph_cluster') }}" + +- name: Cleaning up temp file on localhost + local_action: file path=/tmp/kolla_ceph_cluster state=absent + changed_when: False + always_run: True + run_once: True + +- name: Starting Ceph Monitor data container + docker: + docker_api_version: "{{ docker_api_version }}" + net: host + pull: "{{ docker_pull_policy }}" + restart_policy: "{{ docker_restart_policy }}" + restart_policy_retry: "{{ docker_restart_policy_retry }}" + state: reloaded + registry: "{{ docker_registry }}" + username: "{{ docker_registry_username }}" + password: "{{ docker_registry_password }}" + insecure_registry: "{{ docker_insecure_registry }}" + name: ceph_mon_data + image: "{{ ceph_data_image_full }}" + volumes: + - "/etc/ceph/" + - "/var/lib/ceph/" + +- include: generate_cluster.yml + when: delegate_host == 'None' and inventory_hostname == groups['ceph-mon'][0] diff --git a/ansible/roles/ceph/tasks/bootstrap_osds.yml b/ansible/roles/ceph/tasks/bootstrap_osds.yml new file mode 100644 index 0000000000..59163f8d39 --- /dev/null +++ b/ansible/roles/ceph/tasks/bootstrap_osds.yml @@ -0,0 +1,48 @@ +--- +- name: Looking up disks to bootstrap for Ceph + find_disks: + partition_name: 'KOLLA_CEPH_OSD_BOOTSTRAP' + register: osds_bootstrap + when: inventory_hostname in groups['ceph-osd'] + +- name: Bootstrapping Ceph OSDs + docker: + docker_api_version: "{{ docker_api_version }}" + net: host + pull: "{{ docker_pull_policy }}" + restart_policy: "no" + state: reloaded + registry: "{{ docker_registry }}" + username: "{{ docker_registry_username }}" + password: "{{ docker_registry_password }}" + insecure_registry: "{{ docker_insecure_registry }}" + privileged: True + name: "bootstrap_osd_{{ item.0 }}" + image: "{{ ceph_osd_image_full }}" + volumes: + - "{{ node_config_directory }}/ceph-osd/:/opt/kolla/ceph-osd/:ro" + - "/dev/:/dev/" + env: + KOLLA_BOOTSTRAP: + KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" + OSD_DEV: "{{ item.1.device }}" + OSD_INITIAL_WEIGHT: "{{ osd_initial_weight }}" + with_indexed_items: osds_bootstrap['disks']|default([]) + when: inventory_hostname in groups['ceph-osd'] + +# https://github.com/ansible/ansible-modules-core/pull/1031 +- name: Waiting for bootstrap containers to exit + command: docker wait "bootstrap_osd_{{ item.0 }}" + register: bootstrap_result + run_once: True + failed_when: bootstrap_result.stdout != "0" + with_indexed_items: osds_bootstrap['disks']|default([]) + when: inventory_hostname in groups['ceph-osd'] + +- name: Cleaning up bootstrap containers + docker: + name: "bootstrap_osd_{{ item.0 }}" + image: "{{ ceph_osd_image_full }}" + state: absent + with_indexed_items: osds_bootstrap['disks']|default([]) + when: inventory_hostname in groups['ceph-osd'] diff --git a/ansible/roles/ceph/tasks/config.yml b/ansible/roles/ceph/tasks/config.yml new file mode 100644 index 0000000000..d242ea80c8 --- /dev/null +++ b/ansible/roles/ceph/tasks/config.yml @@ -0,0 +1,22 @@ +--- +- include: ../../config.yml + vars: + service_name: "ceph-mon" + config_source: + - "roles/ceph/templates/ceph.conf.j2" + - "/etc/kolla/config/ceph.conf" + config_template_dest: + - "{{ node_templates_directory }}/{{ service_name }}/ceph.conf_minimal" + - "{{ node_templates_directory }}/{{ service_name }}/ceph.conf_augment" + config_dest: "{{ node_config_directory }}/{{ service_name }}/ceph.conf" + +- include: ../../config.yml + vars: + service_name: "ceph-osd" + config_source: + - "roles/ceph/templates/ceph.conf.j2" + - "/etc/kolla/config/ceph.conf" + config_template_dest: + - "{{ node_templates_directory }}/{{ service_name }}/ceph.conf_minimal" + - "{{ node_templates_directory }}/{{ service_name }}/ceph.conf_augment" + config_dest: "{{ node_config_directory }}/{{ service_name }}/ceph.conf" diff --git a/ansible/roles/ceph/tasks/distribute_keyrings.yml b/ansible/roles/ceph/tasks/distribute_keyrings.yml new file mode 100644 index 0000000000..10ec0a8ddf --- /dev/null +++ b/ansible/roles/ceph/tasks/distribute_keyrings.yml @@ -0,0 +1,34 @@ +--- +- name: Fetching Ceph keyrings + command: docker exec ceph_mon fetch_ceph_keys.py + register: ceph_files_json + changed_when: "{{ (ceph_files_json.stdout | from_json).changed }}" + failed_when: "{{ (ceph_files_json.stdout | from_json).failed }}" + delegate_to: "{{ delegate_host }}" + run_once: True + +- name: Reading json from variable + set_fact: + ceph_files: "{{ (ceph_files_json.stdout | from_json) }}" + +- name: Pushing Ceph keyring for OSDs + bslurp: + src: "{{ item.content }}" + dest: "{{ node_config_directory }}/ceph-osd/{{ item.filename }}" + mode: 0600 + sha1: "{{ item.sha1 }}" + with_items: + - "{{ ceph_files['ceph.client.admin.keyring'] }}" + when: inventory_hostname in groups['ceph-osd'] + +- name: Pushing Ceph keyrings for Mons + bslurp: + src: "{{ item.content }}" + dest: "{{ node_config_directory }}/ceph-mon/{{ item.filename }}" + mode: 0600 + sha1: "{{ item.sha1 }}" + with_items: + - "{{ ceph_files['ceph.client.admin.keyring'] }}" + - "{{ ceph_files['ceph.client.mon.keyring'] }}" + - "{{ ceph_files['ceph.monmap'] }}" + when: inventory_hostname in groups['ceph-mon'] diff --git a/ansible/roles/ceph/tasks/generate_cluster.yml b/ansible/roles/ceph/tasks/generate_cluster.yml new file mode 100644 index 0000000000..1a0f16b1b8 --- /dev/null +++ b/ansible/roles/ceph/tasks/generate_cluster.yml @@ -0,0 +1,30 @@ +--- +- name: Generating Initial Ceph keyrings and monmap + docker: + detach: False + docker_api_version: "{{ docker_api_version }}" + net: host + pull: "{{ docker_pull_policy }}" + restart_policy: "no" + state: reloaded + registry: "{{ docker_registry }}" + username: "{{ docker_registry_username }}" + password: "{{ docker_registry_password }}" + insecure_registry: "{{ docker_insecure_registry }}" + name: ceph_mon + image: "{{ ceph_mon_image_full }}" + volumes: "{{ node_config_directory }}/ceph-mon/:/opt/kolla/ceph-mon/:ro" + volumes_from: + - "ceph_mon_data" + env: + KOLLA_BOOTSTRAP: + KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" + MON_IP: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}" + +- name: Waiting for a few settings for cluster to generate keys + command: sleep 3 + changed_when: False + +- name: Setting host for cluster files + set_fact: + delegate_host: "{{ ansible_hostname }}" diff --git a/ansible/roles/ceph/tasks/main.yml b/ansible/roles/ceph/tasks/main.yml new file mode 100644 index 0000000000..63dca3460f --- /dev/null +++ b/ansible/roles/ceph/tasks/main.yml @@ -0,0 +1,16 @@ +--- +- include: config.yml + +- include: bootstrap_mons.yml + when: inventory_hostname in groups['ceph-mon'] + +- include: distribute_keyrings.yml + +- include: start_mons.yml + when: inventory_hostname in groups['ceph-mon'] + +- include: bootstrap_osds.yml + when: inventory_hostname in groups['ceph-osd'] + +- include: start_osds.yml + when: inventory_hostname in groups['ceph-osd'] diff --git a/ansible/roles/ceph/tasks/start_mons.yml b/ansible/roles/ceph/tasks/start_mons.yml new file mode 100644 index 0000000000..ab50d73554 --- /dev/null +++ b/ansible/roles/ceph/tasks/start_mons.yml @@ -0,0 +1,22 @@ +--- +- name: Starting ceph-mon container + docker: + docker_api_version: "{{ docker_api_version }}" + net: host + pull: "{{ docker_pull_policy }}" + restart_policy: "{{ docker_restart_policy }}" + restart_policy_retry: "{{ docker_restart_policy_retry }}" + state: reloaded + registry: "{{ docker_registry }}" + username: "{{ docker_registry_username }}" + password: "{{ docker_registry_password }}" + insecure_registry: "{{ docker_insecure_registry }}" + name: ceph_mon + image: "{{ ceph_mon_image_full }}" + volumes: "{{ node_config_directory }}/ceph-mon/:/opt/kolla/ceph-mon/:ro" + volumes_from: + - "ceph_mon_data" + env: + KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" + MON_IP: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}" + when: inventory_hostname in groups['ceph-mon'] diff --git a/ansible/roles/ceph/tasks/start_osds.yml b/ansible/roles/ceph/tasks/start_osds.yml new file mode 100644 index 0000000000..593ec7b15b --- /dev/null +++ b/ansible/roles/ceph/tasks/start_osds.yml @@ -0,0 +1,50 @@ +--- +- name: Looking up OSDs for Ceph + find_disks: + partition_name: 'KOLLA_CEPH_DATA' + register: osds + +- name: Mounting Ceph OSD volumes + mount: + src: "UUID={{ item.fs_uuid }}" + fstype: xfs + state: mounted + name: "/var/lib/ceph/osd/{{ item.fs_uuid }}" + with_items: osds.disks + +- name: Gathering OSD IDs + command: 'cat /var/lib/ceph/osd/{{ item.fs_uuid }}/whoami' + with_items: osds.disks + register: id + changed_when: False + failed_when: id.rc != 0 + +- name: Starting ceph-osds container + docker: + docker_api_version: "{{ docker_api_version }}" + net: host + pull: "{{ docker_pull_policy }}" + restart_policy: "{{ docker_restart_policy }}" + restart_policy_retry: "{{ docker_restart_policy_retry }}" + state: reloaded + registry: "{{ docker_registry }}" + username: "{{ docker_registry_username }}" + password: "{{ docker_registry_password }}" + insecure_registry: "{{ docker_insecure_registry }}" + pid: host + privileged: True + name: "ceph_osd_{{ item.0.stdout }}" + image: "{{ ceph_osd_image_full }}" + volumes: + - "/var/lib/ceph/osd/{{ item.1.fs_uuid }}:/var/lib/ceph/osd/ceph-{{ item.0.stdout }}" + - "{{ node_config_directory }}/ceph-osd/:/opt/kolla/ceph-osd/:ro" + - "/dev/:/dev/" + env: + KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" + OSD_ID: "{{ item.0.stdout }}" + OSD_DEV: "{{ item.1.device }}" + with_together: + - id.results + - osds.disks + when: inventory_hostname in groups['ceph-osd'] + and osds.disks diff --git a/ansible/roles/ceph/templates/ceph.conf.j2 b/ansible/roles/ceph/templates/ceph.conf.j2 new file mode 100644 index 0000000000..9518742d73 --- /dev/null +++ b/ansible/roles/ceph/templates/ceph.conf.j2 @@ -0,0 +1,9 @@ +[global] +fsid = {{ ceph_cluster_fsid }} +mon initial members = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['ansible_hostname'] }}{% if not loop.last %}, {% endif %}{% endfor %} + +mon host = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['ansible_' + api_interface]['ipv4']['address'] }}{% if not loop.last %}, {% endif %}{% endfor %} + +auth cluster required = cephx +auth service required = cephx +auth client required = cephx diff --git a/ansible/site.yml b/ansible/site.yml index 5052000b67..3ba043b8ef 100755 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -1,4 +1,8 @@ --- +- hosts: [ceph-mon, ceph-osd] + roles: + - { role: ceph, tags: ceph, when: enable_ceph | bool } + - hosts: [haproxy, mariadb, rabbitmq, cinder-api, glance-api, keystone, nova-api, neutron-server, swift-proxy-server] roles: - { role: haproxy, tags: haproxy, when: enable_haproxy | bool } diff --git a/docker/ceph/ceph-base/Dockerfile.j2 b/docker/ceph/ceph-base/Dockerfile.j2 index b18890b5d3..363346bce7 100644 --- a/docker/ceph/ceph-base/Dockerfile.j2 +++ b/docker/ceph/ceph-base/Dockerfile.j2 @@ -15,3 +15,7 @@ RUN apt-get install -y --no-install-recommends \ && apt-get clean {% endif %} + +RUN useradd --user-group ceph \ + && mkdir -p /home/ceph \ + && chown -R ceph: /home/ceph diff --git a/docker/ceph/ceph-mon/Dockerfile.j2 b/docker/ceph/ceph-mon/Dockerfile.j2 index e9d47fc708..e8a1337ac9 100644 --- a/docker/ceph/ceph-mon/Dockerfile.j2 +++ b/docker/ceph/ceph-mon/Dockerfile.j2 @@ -3,5 +3,6 @@ MAINTAINER Kolla Project (https://launchpad.net/kolla) COPY start.sh / COPY config-external.sh /opt/kolla/ +COPY fetch_ceph_keys.py /usr/bin/ CMD ["/start.sh"] diff --git a/docker/ceph/ceph-mon/config-external.sh b/docker/ceph/ceph-mon/config-external.sh index 75488239c7..289cffc86b 100644 --- a/docker/ceph/ceph-mon/config-external.sh +++ b/docker/ceph/ceph-mon/config-external.sh @@ -8,3 +8,33 @@ if [[ -f "$SOURCE" ]]; then chown ${OWNER}: $TARGET chmod 0644 $TARGET fi + +SOURCE="/opt/kolla/ceph-mon/ceph.client.admin.keyring" +TARGET="/etc/ceph/ceph.client.admin.keyring" +OWNER="ceph" + +if [[ -f "$SOURCE" ]]; then + cp $SOURCE $TARGET + chown ${OWNER}: $TARGET + chmod 0600 $TARGET +fi + +SOURCE="/opt/kolla/ceph-mon/ceph.client.mon.keyring" +TARGET="/etc/ceph/ceph.client.mon.keyring" +OWNER="ceph" + +if [[ -f "$SOURCE" ]]; then + cp $SOURCE $TARGET + chown ${OWNER}: $TARGET + chmod 0600 $TARGET +fi + +SOURCE="/opt/kolla/ceph-mon/ceph.monmap" +TARGET="/etc/ceph/ceph.monmap" +OWNER="ceph" + +if [[ -f "$SOURCE" ]]; then + cp $SOURCE $TARGET + chown ${OWNER}: $TARGET + chmod 0600 $TARGET +fi diff --git a/docker/ceph/ceph-mon/fetch_ceph_keys.py b/docker/ceph/ceph-mon/fetch_ceph_keys.py new file mode 100755 index 0000000000..ab2be9c16f --- /dev/null +++ b/docker/ceph/ceph-mon/fetch_ceph_keys.py @@ -0,0 +1,67 @@ +#!/usr/bin/python + +# Copyright 2015 Sam Yaple +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a stripped down version of an ansible module I wrote in Yaodu to +# achieve the same goals we have for Kolla. I have relicensed it for Kolla +# https://github.com/SamYaple/yaodu/blob/master/ansible/library/bslurp + +# Basically this module will fetch the admin and mon keyrings as well as the +# monmap file. It then hashes the content, compresses them, and finally it +# converts them to base64 to be safely transported around with ansible + +import base64 +import hashlib +import json +import os +import sys +import zlib + + +def json_exit(msg=None, failed=False, changed=False): + if type(msg) is not dict: + msg = {'msg': str(msg)} + msg.update({'failed': failed, 'changed': changed}) + print(json.dumps(msg)) + sys.exit() + + +def read_file(filename): + filename_path = os.path.join('/etc/ceph', filename) + + if not os.path.exists(filename_path): + json_exit("file not found: {}".format(filename_path), failed=True) + if not os.access(filename_path, os.R_OK): + json_exit("file not readable: {}".format(filename_path), failed=True) + + with open(filename_path, 'rb') as f: + raw_data = f.read() + + return {'content': base64.b64encode(zlib.compress(raw_data)), + 'sha1': hashlib.sha1(raw_data).hexdigest(), + 'filename': filename} + + +def main(): + admin_keyring = 'ceph.client.admin.keyring' + mon_keyring = 'ceph.client.mon.keyring' + monmap = 'ceph.monmap' + + files = [admin_keyring, mon_keyring, monmap] + json_exit({filename: read_file(filename) for filename in files}) + + +if __name__ == '__main__': + main() diff --git a/docker/ceph/ceph-mon/start.sh b/docker/ceph/ceph-mon/start.sh old mode 100644 new mode 100755 index eb5d7f5afd..3afaa3ad40 --- a/docker/ceph/ceph-mon/start.sh +++ b/docker/ceph/ceph-mon/start.sh @@ -3,11 +3,11 @@ set -o errexit CMD="/usr/bin/ceph-mon" -ARGS="-d -i ${MON_NAME} --public-addr ${MON_IP}:6789" +ARGS="-d -i $(hostname) --public-addr ${MON_IP}:6789" # Setup common paths -KEYRING_ADMIN="/etc/ceph/ceph.admin.keyring" -KEYRING_MON="/etc/ceph/ceph.mon.keyring" +KEYRING_ADMIN="/etc/ceph/ceph.client.admin.keyring" +KEYRING_MON="/etc/ceph/ceph.client.mon.keyring" MONMAP="/etc/ceph/ceph.monmap" MON_DIR="/var/lib/ceph/mon/ceph-$(hostname)" @@ -21,7 +21,7 @@ set_configs # of the KOLLA_BOOTSTRAP variable being set, including empty. if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then # Lookup our fsid from the ceph.conf - FSID="$(awk '/^fsid/ {print $3; exit}' ${ceph_conf})" + FSID="$(awk '/^fsid/ {print $3; exit}' /etc/ceph/ceph.conf)" # Generating initial keyrings and monmap ceph-authtool --create-keyring "${KEYRING_MON}" --gen-key -n mon. --cap mon 'allow *' @@ -29,8 +29,8 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then ceph-authtool "${KEYRING_MON}" --import-keyring "${KEYRING_ADMIN}" monmaptool --create --add "$(hostname)" "${MON_IP}" --fsid "${FSID}" "${MONMAP}" - # TODO(SamYaple): Return json parsible output to ansible - exit 0 + echo "Sleeping until keys are fetched" + /bin/sleep infinity fi # This section runs on every mon that does not have a keyring already. diff --git a/docker/ceph/ceph-osd/config-external.sh b/docker/ceph/ceph-osd/config-external.sh index 8e17487edb..8928f91cbd 100644 --- a/docker/ceph/ceph-osd/config-external.sh +++ b/docker/ceph/ceph-osd/config-external.sh @@ -8,3 +8,13 @@ if [[ -f "$SOURCE" ]]; then chown ${OWNER}: $TARGET chmod 0644 $TARGET fi + +SOURCE="/opt/kolla/ceph-osd/ceph.client.admin.keyring" +TARGET="/etc/ceph/ceph.client.admin.keyring" +OWNER="ceph" + +if [[ -f "$SOURCE" ]]; then + cp $SOURCE $TARGET + chown ${OWNER}: $TARGET + chmod 0600 $TARGET +fi diff --git a/docker/ceph/ceph-osd/start.sh b/docker/ceph/ceph-osd/start.sh old mode 100644 new mode 100755 index cf9114a62e..84549ae8c3 --- a/docker/ceph/ceph-osd/start.sh +++ b/docker/ceph/ceph-osd/start.sh @@ -1,9 +1,7 @@ #!/bin/bash +set -o xtrace set -o errexit -CMD="/usr/bin/ceph-osd" -ARGS="-f -d -i ${OSD_ID} --osd-journal ${OSD_DIR}/journal -k ${OSD_DIR}/keyring" - # Loading common functions. source /opt/kolla/kolla-common.sh @@ -13,23 +11,45 @@ set_configs # Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases # of the KOLLA_BOOTSTRAP variable being set, including empty. if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then - # Creating a new label for the disk - parted "${OSD_DEV}" -s -- mklabel gpt + JOURNAL_PARTUUID=$(uuidgen) + + # Formating disk for ceph + sgdisk --zap-all -- "${OSD_DEV}" + sgdisk --new=2:1M:5G --change-name=2:KOLLA_CEPH_JOURNAL --typecode=2:45B0969E-9B03-4F30-B4C6-B4B80CEFF106 --partition-guid=2:${JOURNAL_PARTUUID} --mbrtogpt -- "${OSD_DEV}" + sgdisk --largest-new=1 --change-name=1:KOLLA_CEPH_DATA --typecode=1:4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D -- "${OSD_DEV}" + # This command may throw errors that we can safely ignore + partprobe || true + + # We look up the appropriate device path with partition. + OSD_PARTITION="$(ls ${OSD_DEV}* | egrep ${OSD_DEV}p?1)" + JOURNAL_PARTITION="${OSD_PARTITION%?}2" - # Preparing the OSD for use with Ceph - ceph-disk prepare "${OSD_DEV}" OSD_ID="$(ceph osd create)" OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}" mkdir -p "${OSD_DIR}" - mount "${OSD_DEV}1" "${OSD_DIR}" - ceph-osd -i "${OSD_ID}" --mkfs --mkkey - ceph auth add "osd.${OSD_ID}" osd 'allow *' mon 'allow proflie osd' -i "${OSD_DIR}/keyring" + mkfs.xfs -f "${OSD_PARTITION}" + mount "${OSD_PARTITION}" "${OSD_DIR}" - # Adding osd to crush map + # This will through an error about no key existing. That is normal. It then + # creates the key in the next step. + ceph-osd -i "${OSD_ID}" --mkfs --osd-journal="${JOURNAL_PARTITION}" --mkkey + ceph auth add "osd.${OSD_ID}" osd 'allow *' mon 'allow profile osd' -i "${OSD_DIR}/keyring" + umount "${OSD_PARTITION}" + + # These commands only need to be run once per host but are safe to run + # repeatedly. This can be improved later or if any problems arise. ceph osd crush add-bucket "$(hostname)" host ceph osd crush move "$(hostname)" root=default + + # Adding osd to crush map ceph osd crush add "${OSD_ID}" "${OSD_INITIAL_WEIGHT}" host="$(hostname)" exit 0 fi +# We look up the appropriate journal since we cannot rely on symlinks +JOURNAL_PARTITION="$(ls ${OSD_DEV}* | egrep ${OSD_DEV}p?2)" +OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}" +CMD="/usr/bin/ceph-osd" +ARGS="-f -d -i ${OSD_ID} --osd-journal ${JOURNAL_PARTITION} -k ${OSD_DIR}/keyring" + exec $CMD $ARGS diff --git a/docker/openstack-base/Dockerfile.j2 b/docker/openstack-base/Dockerfile.j2 index 1880512afc..f7dec9e219 100644 --- a/docker/openstack-base/Dockerfile.j2 +++ b/docker/openstack-base/Dockerfile.j2 @@ -40,7 +40,6 @@ RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ python-tuskarclient \ python-zaqarclient \ python-openstackclient \ - MySQL-python \ - numpy + MySQL-python {% endif %} diff --git a/etc/kolla/config/ceph.conf b/etc/kolla/config/ceph.conf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/etc/kolla/passwords.yml b/etc/kolla/passwords.yml index 3d7ec924fa..c2c077b1c5 100644 --- a/etc/kolla/passwords.yml +++ b/etc/kolla/passwords.yml @@ -3,6 +3,12 @@ # Ansible vault for locking down the secrets properly. +################### +# Ceph options +#################### +ceph_cluster_fsid: "5fba2fbc-551d-11e5-a8ce-01ef4c5cf93c" + + ################### # Database options #################### @@ -46,6 +52,7 @@ heat_domain_admin_password: "password" murano_database_password: "password" murano_keystone_password: "password" + #################### # RabbitMQ options ####################