Switch ceph to use storage network

Ceph was not properly using storage network.

TrivialFix

Change-Id: Ibf3da5d19cd2ca874d251b455a7eb856154fc3f7

ansible/roles/ceph/tasks/config.yml

@@ -1,4 +1,13 @@
 ---
+- name: Looking up hostname
+  command: getent hosts {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}
+  changed_when: False
+  register: storage_hostname
+
+- name: Setting storage hostname
+  set_fact:
+    storage_hostname: "{{ storage_hostname.stdout.split()[1] }}"
+
 - name: Ensuring config directories exist
   file:
     path: "{{ node_config_directory }}/{{ item }}"

ansible/roles/ceph/tasks/generate_cluster.yml

@@ -7,7 +7,8 @@
     environment:
       KOLLA_BOOTSTRAP:
       KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
-      MON_IP: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
+      MON_IP: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}"
+      HOSTNAME: "{{ storage_hostname }}"
     image: "{{ ceph_mon_image_full }}"
     labels:
       BOOTSTRAP:
@@ -18,16 +19,7 @@
       - "ceph_mon:/var/lib/ceph"
       - "ceph_mon_config:/etc/ceph"
 
-- name: Starting ceph-mon container
-  kolla_docker:
-    action: "start_container"
-    common_options: "{{ docker_common_options }}"
-    image: "{{ ceph_mon_image_full }}"
-    name: "ceph_mon"
-    volumes:
-      - "{{ node_config_directory }}/ceph-mon/:{{ container_config_directory }}/:ro"
-      - "ceph_mon:/var/lib/ceph"
-      - "ceph_mon_config:/etc/ceph"
+- include: start_mons.yml
 
 - name: Setting host for cluster files
   set_fact:

ansible/roles/ceph/tasks/start_mons.yml

@@ -3,6 +3,9 @@
   kolla_docker:
     action: "start_container"
     common_options: "{{ docker_common_options }}"
+    environment:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+      HOSTNAME: "{{ storage_hostname }}"
     image: "{{ ceph_mon_image_full }}"
     name: "ceph_mon"
     volumes:

ansible/roles/ceph/templates/ceph-mon.json.j2

@@ -1,5 +1,5 @@
 {
-    "command": "/usr/bin/ceph-mon -d -i {{ ansible_hostname }} --public-addr {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}:6789",
+    "command": "/usr/bin/ceph-mon -d -i {{ storage_hostname }} --public-addr {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}:6789",
     "config_files": [
         {
             "source": "{{ container_config_directory }}/ceph.conf",

ansible/roles/ceph/templates/ceph.conf.j2

@@ -1,8 +1,10 @@
 [global]
 fsid = {{ ceph_cluster_fsid }}
-mon initial members = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['ansible_hostname'] }}{% if not loop.last %}, {% endif %}{% endfor %}
+mon initial members = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['storage_hostname'] }}{% if not loop.last %}, {% endif %}{% endfor %}
 
-mon host = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}{% if not loop.last %}, {% endif %}{% endfor %}
+mon host = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['storage_hostname'] }}{% if not loop.last %}, {% endif %}{% endfor %}
+
+mon addr = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['ansible_' + hostvars[host]['storage_interface']]['ipv4']['address'] }}:6789{% if not loop.last %}, {% endif %}{% endfor %}
 
 auth cluster required = cephx
 auth service required = cephx
@@ -10,8 +12,8 @@ auth client required = cephx
 
 {% if service_name is defined and service_name == 'ceph-rgw' %}
 [client.radosgw.gateway]
-host = {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}
+host = {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}
 keyring = /etc/ceph/ceph.client.radosgw.keyring
 log file = /var/log/radosgw/client.radosgw.gateway.log
-rgw frontends = civetweb port={{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}:{{ rgw_port }}
+rgw frontends = civetweb port={{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}:{{ rgw_port }}
 {% endif %}

docker/ceph/ceph-mon/extend_start.sh

@@ -5,7 +5,7 @@ KEYRING_ADMIN="/etc/ceph/ceph.client.admin.keyring"
 KEYRING_MON="/etc/ceph/ceph.client.mon.keyring"
 KEYRING_RGW="/etc/ceph/ceph.client.radosgw.keyring"
 MONMAP="/etc/ceph/ceph.monmap"
-MON_DIR="/var/lib/ceph/mon/ceph-$(hostname -s)"
+MON_DIR="/var/lib/ceph/mon/ceph-${HOSTNAME}"
 
 # Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases
 # of the KOLLA_BOOTSTRAP variable being set, including empty.
@@ -19,7 +19,7 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
     ceph-authtool --create-keyring "${KEYRING_RGW}" --gen-key -n client.radosgw.gateway --set-uid=0 --cap osd 'allow rwx' --cap mon 'allow rwx'
     ceph-authtool "${KEYRING_MON}" --import-keyring "${KEYRING_ADMIN}"
     ceph-authtool "${KEYRING_MON}" --import-keyring "${KEYRING_RGW}"
-    monmaptool --create --add "$(hostname -s)" "${MON_IP}" --fsid "${FSID}" "${MONMAP}"
+    monmaptool --create --add "${HOSTNAME}" "${MON_IP}" --fsid "${FSID}" "${MONMAP}"
 
     exit 0
 fi
@@ -32,6 +32,6 @@ if [[ ! -e "${MON_DIR}/keyring" ]]; then
     ceph-authtool --create-keyring "${KEYRING_TMP}" --import-keyring "${KEYRING_ADMIN}"
     ceph-authtool "${KEYRING_TMP}" --import-keyring "${KEYRING_MON}"
     mkdir -p "${MON_DIR}"
-    ceph-mon --mkfs -i "$(hostname -s)" --monmap "${MONMAP}" --keyring "${KEYRING_TMP}"
+    ceph-mon --mkfs -i "${HOSTNAME}" --monmap "${MONMAP}" --keyring "${KEYRING_TMP}"
     rm "${KEYRING_TMP}"
 fi