Allow Kolla Ceph to deploy bluestore OSDs in Kolla-ansible

Support Kolla Ceph to deploy blustore OSDs with Kolla-ansible.

Please refer to [1] for bluestore OSD configuration

The patch includes:
1. Set Ceph OSD store type group_vars/all.yml. The default value
is "bluestore" in Rocky.

2. Make Kolla Ceph to deploy bluestore OSDs with Kolla-ansible

3. Update gate test configuration for Ceph bluestore OSD test

[1]: specs/kolla-ceph-bluestore.rst

Partially-Implements: blueprint kolla-ceph-bluestore
Depends-On: I00eaa600a5e9ad4c1ebca2eeb523bca3d7a25128
Change-Id: I14f20a00654dff32c36d078ebb9005d91a3e60b2
Signed-off-by: Tone Zhang <tone.zhang@arm.com>

ansible/group_vars/all.yml

@@ -768,6 +768,10 @@ ceph_cache_rule: "cache host firstn"
 ceph_pool_pg_num: 8
 ceph_pool_pgp_num: 8
 
+# Set the store type for ceph OSD
+# Valid options are [ filestore, bluestore]
+ceph_osd_store_type: "bluestore"
+
 #####################
 # VMware support
 ######################

ansible/roles/ceph/defaults/main.yml

@@ -46,6 +46,10 @@ ceph_client_admin_keyring_caps:
   osd: "allow *"
   mgr: "allow *"
 
+partition_name_osd_bootstrap: "{{ 'KOLLA_CEPH_OSD_BOOTSTRAP_BS' if ceph_osd_store_type == 'bluestore' else 'KOLLA_CEPH_OSD_BOOTSTRAP' }}"
+partition_name_cache_bootstrap: "{{ 'KOLLA_CEPH_OSD_CACHE_BOOTSTRAP_BS' if ceph_osd_store_type == 'bluestore' else 'KOLLA_CEPH_OSD_CACHE_BOOTSTRAP' }}"
+partition_name_osd_data: "{{ 'KOLLA_CEPH_BSDATA' if ceph_osd_store_type == 'bluestore' else 'KOLLA_CEPH_DATA' }}"
+
 ####################
 ## Ceph_rgw_keystone
 ####################

ansible/roles/ceph/tasks/bootstrap_osds.yml

@@ -2,7 +2,7 @@
 - name: Looking up disks to bootstrap for Ceph OSDs
   command: docker exec -t kolla_toolbox sudo -E ansible localhost
     -m find_disks
-    -a "partition_name='KOLLA_CEPH_OSD_BOOTSTRAP' match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}"
+    -a "partition_name={{ partition_name_osd_bootstrap }} match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}"
   register: osd_lookup
   changed_when: osd_lookup.stdout.find('localhost | SUCCESS => ') != -1 and (osd_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed
   failed_when: osd_lookup.stdout.split()[2] != 'SUCCESS'
@@ -14,7 +14,7 @@
 - name: Looking up disks to bootstrap for Ceph Cache OSDs
   command: docker exec -t kolla_toolbox sudo -E ansible localhost
     -m find_disks
-    -a "partition_name='KOLLA_CEPH_OSD_CACHE_BOOTSTRAP' match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}"
+    -a "partition_name={{ partition_name_cache_bootstrap }} match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}"
   register: osd_cache_lookup
   changed_when: osd_cache_lookup.stdout.find('localhost | SUCCESS => ') != -1 and (osd_cache_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed
   failed_when: osd_cache_lookup.stdout.split()[2] != 'SUCCESS'
@@ -58,6 +58,19 @@
       OSD_FILESYSTEM: "{{ ceph_osd_filesystem }}"
       OSD_INITIAL_WEIGHT: "{{ osd_initial_weight }}"
       HOSTNAME: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}"
+      OSD_STORETYPE: "{{ ceph_osd_store_type }}"
+      OSD_BS_DEV: "{{ item.1.device | default('') }}"
+      OSD_BS_LABEL: "{{ item.1.partition_label | default('') }}"
+      OSD_BS_PARTNUM: "{{ item.1.partition_num | default('') }}"
+      OSD_BS_BLK_DEV: "{{ item.1.bs_blk_device | default('') }}"
+      OSD_BS_BLK_LABEL: "{{ item.1.bs_blk_label | default('') }}"
+      OSD_BS_BLK_PARTNUM: "{{ item.1.bs_blk_partition_num | default('') }}"
+      OSD_BS_WAL_DEV: "{{ item.1.bs_wal_device | default('') }}"
+      OSD_BS_WAL_LABEL: "{{ item.1.bs_wal_label | default('') }}"
+      OSD_BS_WAL_PARTNUM: "{{ item.1.bs_wal_partition_num | default('') }}"
+      OSD_BS_DB_DEV: "{{ item.1.bs_db_device | default('') }}"
+      OSD_BS_DB_LABEL: "{{ item.1.bs_db_label | default('') }}"
+      OSD_BS_DB_PARTNUM: "{{ item.1.bs_db_partition_num | default('') }}"
     image: "{{ ceph_osd_image_full }}"
     labels:
       BOOTSTRAP:
@@ -106,6 +119,19 @@
       OSD_FILESYSTEM: "{{ ceph_osd_filesystem }}"
       OSD_INITIAL_WEIGHT: "{{ osd_initial_weight }}"
       HOSTNAME: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}"
+      OSD_STORETYPE: "{{ ceph_osd_store_type }}"
+      OSD_BS_DEV: "{{ item.1.device | default('') }}"
+      OSD_BS_LABEL: "{{ item.1.partition_label | default('') }}"
+      OSD_BS_PARTNUM: "{{ item.1.partition_num | default('') }}"
+      OSD_BS_BLK_DEV: "{{ item.1.bs_blk_device | default('') }}"
+      OSD_BS_BLK_LABEL: "{{ item.1.bs_blk_label | default('') }}"
+      OSD_BS_BLK_PARTNUM: "{{ item.1.bs_blk_partition_num | default('') }}"
+      OSD_BS_WAL_DEV: "{{ item.1.bs_wal_device | default('') }}"
+      OSD_BS_WAL_LABEL: "{{ item.1.bs_wal_label|default('') }}"
+      OSD_BS_WAL_PARTNUM: "{{ item.1.bs_wal_partition_num | default('') }}"
+      OSD_BS_DB_DEV: "{{ item.1.bs_db_device | default('') }}"
+      OSD_BS_DB_LABEL: "{{ item.1.bs_db_label | default('') }}"
+      OSD_BS_DB_PARTNUM: "{{ item.1.bs_db_partition_num | default('') }}"
     image: "{{ ceph_osd_image_full }}"
     labels:
       BOOTSTRAP:

ansible/roles/ceph/tasks/reconfigure.yml

@@ -13,7 +13,7 @@
 - name: Looking up OSDs for Ceph
   command: docker exec -t kolla_toolbox sudo -E ansible localhost
     -m find_disks
-    -a "partition_name='KOLLA_CEPH_DATA' match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}"
+    -a "partition_name={{ partition_name_osd_data }} match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}"
   register: osd_lookup
   changed_when: osd_lookup.stdout.find('localhost | SUCCESS => ') != -1 and (osd_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed
   failed_when: osd_lookup.stdout.split()[2] != 'SUCCESS'

ansible/roles/ceph/tasks/start_osds.yml

@@ -2,7 +2,7 @@
 - name: Looking up OSDs for Ceph
   command: docker exec -t kolla_toolbox sudo -E ansible localhost
     -m find_disks
-    -a "partition_name='KOLLA_CEPH_DATA' match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}"
+    -a "partition_name={{ partition_name_osd_data }} match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}"
   register: osd_lookup
   changed_when: osd_lookup.stdout.find('localhost | SUCCESS => ') != -1 and (osd_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed
   failed_when: osd_lookup.stdout.split()[2] != 'SUCCESS'
@@ -40,6 +40,8 @@
       OSD_ID: "{{ item.0.stdout }}"
       JOURNAL_PARTITION: "{{ item.1.journal }}"
       TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES: "{{ ceph_tcmalloc_tc_bytes }}"
+      OSD_STORETYPE: "{{ ceph_osd_store_type }}"
+      OSD_BS_FSUUID: "{{ item.1['fs_uuid'] }}"
     image: "{{ ceph_osd_image_full }}"
     name: "ceph_osd_{{ item.0.stdout }}"
     pid_mode: "host"

releasenotes/notes/kolla-ceph-bluestore-b48673a85dda06d1.yaml

@@ -0,0 +1,6 @@
+---
+prelude: >
+    Since Ceph Luminous release, bluestore OSDs are recommended. Kolla Ceph
+    currently only supports filestore. Bluestore is required in Kolla Ceph.
+features:
+  - Support Kolla Ceph to deploy bluestore OSDs in Rocky release.

tests/setup_ceph_disks.sh

@@ -1,13 +1,23 @@
 mkdir -p /opt/data/kolla
-dd if=/dev/zero of=/opt/data/kolla/ceph-osd0.img bs=5M count=2048
+dd if=/dev/zero of=/opt/data/kolla/ceph-osd0.img bs=5M count=128
 LOOP=$(losetup -f)
 losetup $LOOP /opt/data/kolla/ceph-osd0.img
-parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_OSD1 1 -1
+parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_BS_OSD1 1 -1
 
-dd if=/dev/zero of=/opt/data/kolla/ceph-journal0.img bs=5M count=512
+dd if=/dev/zero of=/opt/data/kolla/ceph-osd0-b.img bs=5M count=2048
 LOOP=$(losetup -f)
-losetup $LOOP /opt/data/kolla/ceph-journal0.img
+losetup $LOOP /opt/data/kolla/ceph-osd0-b.img
-parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_OSD1_J 1 -1
+parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_BS_B_OSD1 1 -1
 
-partprobe $LOOP
+dd if=/dev/zero of=/opt/data/kolla/ceph-osd0-w.img bs=5M count=256
+LOOP=$(losetup -f)
+losetup $LOOP /opt/data/kolla/ceph-osd0-w.img
+parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_BS_W_OSD1 1 -1
+
+dd if=/dev/zero of=/opt/data/kolla/ceph-osd0-d.img bs=5M count=256
+LOOP=$(losetup -f)
+losetup $LOOP /opt/data/kolla/ceph-osd0-d.img
+parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_BS_D_OSD1 1 -1
+
+partprobe