Add os-is-bootstrap-host element and script

As a very temporary master election we are predetermining a host to be
used as the 'bootstrap host'. Also changing mysql, rabbitmq, and
reset-db to use this script.

This requires Change-Id: I85399019c5fc448e98362ef832988abc8d9d459d

Change-Id: I1369926eb428ee1e3e4af861dc76ffe82a07f629

elements/boot-stack/os-refresh-config/configure.d/52-init-openstack

@@ -3,7 +3,7 @@ set -eu
 
 OK=/mnt/state/var/lib/boot-stack/init-openstack.ok
 
-if [ -e $OK ] ; then
+if ! os-is-bootstrap-host || [ -e $OK ] ; then
     exit 0
 fi
 

elements/mysql/element-deps

@@ -1,2 +1,3 @@
 mysql-common
+os-is-bootstrap-host
 os-svc-install

elements/mysql/os-refresh-config/configure.d/51-mysql-init

@@ -1,8 +1,6 @@
 #!/bin/bash
 set -eu
 
-BOOTSTRAP_NODE=$(os-apply-config --key mysql.nodes --type raw --key-default '' | sed 's/,/\n/g' | sort | head -n 1)
-MY_HOST=$(os-apply-config --key mysql.host --type netaddress  --key-default '')
 MYSQL_INITIALIZED="/mnt/state/var/lib/mysql/galera.initialized"
 
 pid_path=/var/run/mysqld/mysqld.pid
@@ -10,16 +8,20 @@ mkdir -p $(dirname $pid_path)
 chown mysql:root $(dirname $pid_path)
 
 if [ ! -e ${MYSQL_INITIALIZED} ]; then
-    if [ "$BOOTSTRAP_NODE" == "$MY_HOST" ]; then
-        # Needed to setup initial tables. This command is idempotent.
+    if os-is-bootstrap-host; then
+       # Needed to setup initial tables. This command is idempotent.
         /usr/local/mysql/scripts/mysql_install_db --user=mysql --basedir=/usr/local/mysql --datadir=/mnt/state/var/lib/mysql --no-defaults --pid-file=$pid_path  --wsrep-new-cluster
 
         # We install this init script so we can trust this path exists
         /etc/init.d/mysql bootstrap-pxc
-    else
-        os-svc-restart -n mysql
-    fi
 
-    # We did db initialization by hand so reset-db doesnt need to interfere
-    touch ${MYSQL_INITIALIZED}
+        touch ${MYSQL_INITIALIZED}
+    elif [ $? -eq 1 ]; then
+        os-svc-restart -n mysql
+
+        touch ${MYSQL_INITIALIZED}
+    else
+        echo "We are neither cluster initializer or joiner. Refusing to bootstrap mysql cluster until role is known."
+        exit 1
+    fi
 fi

elements/os-is-bootstrap-host/README.md

@@ -0,0 +1,23 @@
+A pre-determined master for initial bootstrapping
+
+Configuration
+-------------
+
+For initial master election, `bootstrap_nodeid` and `node_id` should be set
+to the ID string of the node which will be used as initial master and the id
+string of the current node.
+
+A host where os-is-bootstrap-host is true (has exit value and output of 0)
+
+    bootstrap_host:
+      bootstrap_nodeid: "SomeNode1"
+      nodeid: "SomeNode1"
+
+A host where os-is-bootstrap-host is false (has exit value and output of 1)
+
+    bootstrap_host:
+      bootstrap_nodeid: "SomeNode1"
+      nodeid: "SomeNode2"
+
+If either of these values is undefined, the exit value and output of the script
+will be 255.

elements/os-is-bootstrap-host/bin/os-is-bootstrap-host

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -eu
+set -o pipefail
+
+# Used to determine if this host is the "bootstrap host"
+#
+# This is our very temporary method for master election - there should only be
+# one bootstrap host throughout the cluster.
+
+BOOTSTRAP_NODE=$(os-apply-config --key bootstrap_host.bootstrap_nodeid --type netaddress --key-default '')
+MY_HOST=$(os-apply-config --key bootstrap_host.nodeid --type netaddress --key-default '')
+
+# Signal we are not bootstrap now but we could be in future
+if [ -z "$BOOTSTRAP_NODE" -o -z "$MY_HOST" ]; then
+    exit 255
+elif [ "$BOOTSTRAP_NODE" != "$MY_HOST" ]; then
+    exit 1
+fi

elements/rabbitmq-server/element-deps

@@ -1,4 +1,5 @@
 iptables
 os-apply-config
+os-is-bootstrap-host
 os-refresh-config
 os-svc-install

elements/rabbitmq-server/os-refresh-config/post-configure.d/40-rabbitmq

@@ -57,22 +57,22 @@ NODES=${NODES,,}
 MASTER=${MASTER,,}
 
 if [ -n "$NODES" ];then
-    if [ "$MASTER" = "$LOCAL" ];then
+    if os-is-bootstrap-host; then
         # if this is master node which is already clustered, do nothing
-    if is_in_cluster $LOCAL;then
-        exit 0
-    fi
+        if is_in_cluster $LOCAL; then
+            exit 0
+        fi
     else
         # if this node is already in cluster with current master node, do nothing
-        if rabbitmqctl cluster_status|grep -q "$MASTER";then
-        exit 0
-    fi
+        if rabbitmqctl cluster_status|grep -q "$MASTER"; then
+            exit 0
+        fi
     fi
 
     JOINED_WITH=''
     # find another node which is already clustered and try join with it
     for NODE in $NODES;do
-        if [ ! "$NODE" = "$LOCAL" ] && is_in_cluster $NODE;then
+        if [ ! "$NODE" = "$LOCAL" ] && is_in_cluster $NODE; then
             if join_with $NODE; then
                 JOINED_WITH=$NODE
                 break
@@ -80,10 +80,10 @@ if [ -n "$NODES" ];then
         fi
     done
 
-    if [ -z "$JOINED_WITH"];then
+    if [ -z "$JOINED_WITH"]; then
         # if there is no existing cluster yet and this is master node, start this
         # node standalone (other nodes will join to this one)
-        if [ "$MASTER" == "$LOCAL" ];then
+        if os-is-bootstrap-host; then
             rabbitmqctl start_app
         else
             if ! join_with $MASTER; then

elements/seed-stack-config/config.json

@@ -55,6 +55,10 @@
     "public_interface_ip": "192.0.2.1/24",
     "masquerade_networks": ["192.0.2.0/24"]
   },
+  "bootstrap_host": {
+    "bootstrap_nodeid": "seed",
+    "nodeid": "seed"
+  },
   "cinder": {
     "db": "mysql://cinder:unset@localhost/cinder",
     "volume_size_mb": "5000",