diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 10b29c889b..10a1f33490 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -768,6 +768,10 @@ ceph_cache_rule: "cache host firstn" ceph_pool_pg_num: 8 ceph_pool_pgp_num: 8 +# Set the store type for ceph OSD +# Valid options are [ filestore, bluestore] +ceph_osd_store_type: "bluestore" + ##################### # VMware support ###################### diff --git a/ansible/roles/ceph/defaults/main.yml b/ansible/roles/ceph/defaults/main.yml index 851cfdbd5f..6c312da401 100644 --- a/ansible/roles/ceph/defaults/main.yml +++ b/ansible/roles/ceph/defaults/main.yml @@ -46,6 +46,10 @@ ceph_client_admin_keyring_caps: osd: "allow *" mgr: "allow *" +partition_name_osd_bootstrap: "{{ 'KOLLA_CEPH_OSD_BOOTSTRAP_BS' if ceph_osd_store_type == 'bluestore' else 'KOLLA_CEPH_OSD_BOOTSTRAP' }}" +partition_name_cache_bootstrap: "{{ 'KOLLA_CEPH_OSD_CACHE_BOOTSTRAP_BS' if ceph_osd_store_type == 'bluestore' else 'KOLLA_CEPH_OSD_CACHE_BOOTSTRAP' }}" +partition_name_osd_data: "{{ 'KOLLA_CEPH_BSDATA' if ceph_osd_store_type == 'bluestore' else 'KOLLA_CEPH_DATA' }}" + #################### ## Ceph_rgw_keystone #################### diff --git a/ansible/roles/ceph/tasks/bootstrap_osds.yml b/ansible/roles/ceph/tasks/bootstrap_osds.yml index ff80dfc501..8a1c842e8e 100644 --- a/ansible/roles/ceph/tasks/bootstrap_osds.yml +++ b/ansible/roles/ceph/tasks/bootstrap_osds.yml @@ -2,7 +2,7 @@ - name: Looking up disks to bootstrap for Ceph OSDs command: docker exec -t kolla_toolbox sudo -E ansible localhost -m find_disks - -a "partition_name='KOLLA_CEPH_OSD_BOOTSTRAP' match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}" + -a "partition_name={{ partition_name_osd_bootstrap }} match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}" register: osd_lookup changed_when: osd_lookup.stdout.find('localhost | SUCCESS => ') != -1 and (osd_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed failed_when: osd_lookup.stdout.split()[2] != 'SUCCESS' @@ -14,7 +14,7 @@ - name: Looking up disks to bootstrap for Ceph Cache OSDs command: docker exec -t kolla_toolbox sudo -E ansible localhost -m find_disks - -a "partition_name='KOLLA_CEPH_OSD_CACHE_BOOTSTRAP' match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}" + -a "partition_name={{ partition_name_cache_bootstrap }} match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}" register: osd_cache_lookup changed_when: osd_cache_lookup.stdout.find('localhost | SUCCESS => ') != -1 and (osd_cache_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed failed_when: osd_cache_lookup.stdout.split()[2] != 'SUCCESS' @@ -58,6 +58,19 @@ OSD_FILESYSTEM: "{{ ceph_osd_filesystem }}" OSD_INITIAL_WEIGHT: "{{ osd_initial_weight }}" HOSTNAME: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}" + OSD_STORETYPE: "{{ ceph_osd_store_type }}" + OSD_BS_DEV: "{{ item.1.device | default('') }}" + OSD_BS_LABEL: "{{ item.1.partition_label | default('') }}" + OSD_BS_PARTNUM: "{{ item.1.partition_num | default('') }}" + OSD_BS_BLK_DEV: "{{ item.1.bs_blk_device | default('') }}" + OSD_BS_BLK_LABEL: "{{ item.1.bs_blk_label | default('') }}" + OSD_BS_BLK_PARTNUM: "{{ item.1.bs_blk_partition_num | default('') }}" + OSD_BS_WAL_DEV: "{{ item.1.bs_wal_device | default('') }}" + OSD_BS_WAL_LABEL: "{{ item.1.bs_wal_label | default('') }}" + OSD_BS_WAL_PARTNUM: "{{ item.1.bs_wal_partition_num | default('') }}" + OSD_BS_DB_DEV: "{{ item.1.bs_db_device | default('') }}" + OSD_BS_DB_LABEL: "{{ item.1.bs_db_label | default('') }}" + OSD_BS_DB_PARTNUM: "{{ item.1.bs_db_partition_num | default('') }}" image: "{{ ceph_osd_image_full }}" labels: BOOTSTRAP: @@ -106,6 +119,19 @@ OSD_FILESYSTEM: "{{ ceph_osd_filesystem }}" OSD_INITIAL_WEIGHT: "{{ osd_initial_weight }}" HOSTNAME: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}" + OSD_STORETYPE: "{{ ceph_osd_store_type }}" + OSD_BS_DEV: "{{ item.1.device | default('') }}" + OSD_BS_LABEL: "{{ item.1.partition_label | default('') }}" + OSD_BS_PARTNUM: "{{ item.1.partition_num | default('') }}" + OSD_BS_BLK_DEV: "{{ item.1.bs_blk_device | default('') }}" + OSD_BS_BLK_LABEL: "{{ item.1.bs_blk_label | default('') }}" + OSD_BS_BLK_PARTNUM: "{{ item.1.bs_blk_partition_num | default('') }}" + OSD_BS_WAL_DEV: "{{ item.1.bs_wal_device | default('') }}" + OSD_BS_WAL_LABEL: "{{ item.1.bs_wal_label|default('') }}" + OSD_BS_WAL_PARTNUM: "{{ item.1.bs_wal_partition_num | default('') }}" + OSD_BS_DB_DEV: "{{ item.1.bs_db_device | default('') }}" + OSD_BS_DB_LABEL: "{{ item.1.bs_db_label | default('') }}" + OSD_BS_DB_PARTNUM: "{{ item.1.bs_db_partition_num | default('') }}" image: "{{ ceph_osd_image_full }}" labels: BOOTSTRAP: diff --git a/ansible/roles/ceph/tasks/reconfigure.yml b/ansible/roles/ceph/tasks/reconfigure.yml index 0b2e51b5c5..a4832495a9 100644 --- a/ansible/roles/ceph/tasks/reconfigure.yml +++ b/ansible/roles/ceph/tasks/reconfigure.yml @@ -13,7 +13,7 @@ - name: Looking up OSDs for Ceph command: docker exec -t kolla_toolbox sudo -E ansible localhost -m find_disks - -a "partition_name='KOLLA_CEPH_DATA' match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}" + -a "partition_name={{ partition_name_osd_data }} match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}" register: osd_lookup changed_when: osd_lookup.stdout.find('localhost | SUCCESS => ') != -1 and (osd_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed failed_when: osd_lookup.stdout.split()[2] != 'SUCCESS' diff --git a/ansible/roles/ceph/tasks/start_osds.yml b/ansible/roles/ceph/tasks/start_osds.yml index 44fe8292aa..849458b85a 100644 --- a/ansible/roles/ceph/tasks/start_osds.yml +++ b/ansible/roles/ceph/tasks/start_osds.yml @@ -2,7 +2,7 @@ - name: Looking up OSDs for Ceph command: docker exec -t kolla_toolbox sudo -E ansible localhost -m find_disks - -a "partition_name='KOLLA_CEPH_DATA' match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}" + -a "partition_name={{ partition_name_osd_data }} match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}" register: osd_lookup changed_when: osd_lookup.stdout.find('localhost | SUCCESS => ') != -1 and (osd_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed failed_when: osd_lookup.stdout.split()[2] != 'SUCCESS' @@ -40,6 +40,8 @@ OSD_ID: "{{ item.0.stdout }}" JOURNAL_PARTITION: "{{ item.1.journal }}" TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES: "{{ ceph_tcmalloc_tc_bytes }}" + OSD_STORETYPE: "{{ ceph_osd_store_type }}" + OSD_BS_FSUUID: "{{ item.1['fs_uuid'] }}" image: "{{ ceph_osd_image_full }}" name: "ceph_osd_{{ item.0.stdout }}" pid_mode: "host" diff --git a/releasenotes/notes/kolla-ceph-bluestore-b48673a85dda06d1.yaml b/releasenotes/notes/kolla-ceph-bluestore-b48673a85dda06d1.yaml new file mode 100644 index 0000000000..fa14171061 --- /dev/null +++ b/releasenotes/notes/kolla-ceph-bluestore-b48673a85dda06d1.yaml @@ -0,0 +1,6 @@ +--- +prelude: > + Since Ceph Luminous release, bluestore OSDs are recommended. Kolla Ceph + currently only supports filestore. Bluestore is required in Kolla Ceph. +features: + - Support Kolla Ceph to deploy bluestore OSDs in Rocky release. diff --git a/tests/setup_ceph_disks.sh b/tests/setup_ceph_disks.sh index 66a085caa5..582894b8b4 100644 --- a/tests/setup_ceph_disks.sh +++ b/tests/setup_ceph_disks.sh @@ -1,13 +1,23 @@ mkdir -p /opt/data/kolla -dd if=/dev/zero of=/opt/data/kolla/ceph-osd0.img bs=5M count=2048 +dd if=/dev/zero of=/opt/data/kolla/ceph-osd0.img bs=5M count=128 LOOP=$(losetup -f) losetup $LOOP /opt/data/kolla/ceph-osd0.img -parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_OSD1 1 -1 +parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_BS_OSD1 1 -1 -dd if=/dev/zero of=/opt/data/kolla/ceph-journal0.img bs=5M count=512 +dd if=/dev/zero of=/opt/data/kolla/ceph-osd0-b.img bs=5M count=2048 LOOP=$(losetup -f) -losetup $LOOP /opt/data/kolla/ceph-journal0.img -parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_OSD1_J 1 -1 +losetup $LOOP /opt/data/kolla/ceph-osd0-b.img +parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_BS_B_OSD1 1 -1 -partprobe $LOOP +dd if=/dev/zero of=/opt/data/kolla/ceph-osd0-w.img bs=5M count=256 +LOOP=$(losetup -f) +losetup $LOOP /opt/data/kolla/ceph-osd0-w.img +parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_BS_W_OSD1 1 -1 + +dd if=/dev/zero of=/opt/data/kolla/ceph-osd0-d.img bs=5M count=256 +LOOP=$(losetup -f) +losetup $LOOP /opt/data/kolla/ceph-osd0-d.img +parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_BS_D_OSD1 1 -1 + +partprobe