trovestack guide

Also fixed the migration script issue after upgrading SQLAlchemy
version from 1.2.19 to 1.3.4

Change-Id: Ia0d954237dd3c9c2b2ebedf440fa3c656da30777

apidocs/src/samples/db-get-default-instance-configuration-response-json.txt

@@ -1,5 +0,0 @@
-HTTP/1.1 200 OK
-Content-Type: application/json
-Content-Length: 1151
-Date: Mon, 18 Mar 2013 19:09:17 GMT
-

doc/source/admin/index.rst

@@ -10,3 +10,4 @@
    database_module_usage
    guest_cloud_init
    secure_oslo_messaging
+   trovestack

doc/source/admin/trovestack.rst

@@ -0,0 +1,82 @@
+..
+      Copyright 2019 Catalyst IT Ltd
+      All Rights Reserved.
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+Trove integration script - trovestack
+=====================================
+
+``trovestack`` in ``integration/scripts`` folder is a shell script that
+contains lots of useful functionalities via sub-commands including ``install``
+(trove development environment installation), ``unit-tests``,  ``gate-tests``
+(functional test), ``build-image``, etc. This guide introduces some of them.
+
+Before running ``trovestack`` command, go to the scripts folder:
+
+.. code-block:: console
+
+    git clone https://github.com/openstack/trove
+    cd trove/integration/scripts
+
+Build guest agent image
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The trove guest agent image could be created by running the following command:
+
+.. code-block:: console
+
+    $ CONTROLLER_IP=10.0.17.132 \
+        ./trovestack build-image \
+        ${datastore_type} \
+        ${guest_os} \
+        ${guest_os_release} \
+        ${dev_mode}
+
+* Currently, only ``guest_os=ubuntu`` and ``guest_os_release=xenial`` are fully
+  tested.
+
+* ``dev_mode=true`` is mainly for testing purpose for trove developers. When
+  ``dev_mode=true``, ``CONTROLLER_IP`` could be ignored. You need to build the
+  image on the trove controller service host, because the host and the guest VM
+  need to ssh into each other without password. In this mode, when the trove
+  guest agent code is changed, the image doesn't need to be rebuilt which is
+  convenient for debugging. Trove guest agent will ssh into the host and
+  download trove code when the service is initialized.
+
+* if ``dev_mode=false``, the trove code for guest agent is injected into the
+  image at the building time. Now ``dev_mode=false`` is still in experimental
+  and not considered production ready yet.
+
+* If you build the image on host1 but the trove controller service is running
+  on host2, you need to set ``dev_mode=false`` and set ``CONTROLLER_IP`` as the
+  IP address of trove controller service host. As the cloud administrator, you
+  also need to create a Nova keypair and set ``nova_keypair`` option in Trove
+  config file in order to ssh into the guest agent.
+
+For example, build a MySQL image for Ubuntu Xenial operating system:
+
+.. code-block:: console
+
+    $ ./trovestack build-image mysql ubuntu xenial true
+
+Once the image build is finished, the cloud administrator needs to register the
+image in Glance and register a new datastore or version in Trove using
+``trove-manage`` command, e.g. you've built an image for MySQL 5.7.1:
+
+.. code-block:: console
+
+    $ openstack image create ubuntu-mysql-5.7.1-dev \
+      --public \
+      --disk-format qcow2 \
+      --container-format bare \
+      --file ~/images/ubuntu_mysql.qcow2
+    $ trove-manage datastore_version_update mysql 5.7.1 mysql $image_id "" 1

integration/scripts/functions_qemu

@@ -20,13 +20,8 @@ function build_vm() {
     GUEST_CACHEDIR=${GUEST_CACHEDIR:-"$HOME/.cache/image-create"}
     GUEST_WORKING_DIR=${GUEST_WORKING_DIR:-"$HOME/images"}
 
-    export SSH_DIR=${SSH_DIR:-"$HOME/.ssh"}
-    export GUEST_USERNAME=${guest_username}
-    export HOST_SCP_USERNAME=$(whoami)
-    manage_ssh_keys
-
     # In dev mode, the trove guest agent needs to download trove code from
-    # trove-taskmanager host.
+    # trove-taskmanager host during service initialization.
     if [[ "${dev_mode}" == "true" ]]; then
         host_ip=$(ip route get 8.8.8.8 | head -1 | awk '{print $7}')
         export CONTROLLER_IP=${CONTROLLER_IP:-${host_ip}}
@@ -35,7 +30,11 @@ function build_vm() {
         export GUEST_LOGDIR=${GUEST_LOGDIR:-"/var/log/trove/"}
         export ESCAPED_GUEST_LOGDIR=$(echo ${GUEST_LOGDIR} | sed 's/\//\\\//g')
         export TROVESTACK_SCRIPTS=${TROVESTACK_SCRIPTS}
+        export HOST_SCP_USERNAME=$(whoami)
         export HOST_USERNAME=${HOST_SCP_USERNAME}
+        export SSH_DIR=${SSH_DIR:-"$HOME/.ssh"}
+        export GUEST_USERNAME=${guest_username}
+        manage_ssh_keys
     fi
 
     # For system-wide installs, DIB will automatically find the elements, so we only check local path
@@ -58,7 +57,7 @@ function build_vm() {
 
     elementes="$elementes ${guest_os}"
 
-    if [[ "${dev_mode}" != "true" ]]; then
+    if [[ "${dev_mode}" == "false" ]]; then
         elementes="$elementes pip-and-virtualenv"
         elementes="$elementes pip-cache"
         elementes="$elementes no-resolvconf"
@@ -88,21 +87,17 @@ function build_vm() {
 }
 
 function build_guest_image() {
-    exclaim "Building an image for Trove guest agent."
-    datastore_type=$1
+    datastore_type=${1:-"mysql"}
     guest_os=${2:-"ubuntu"}
     guest_release=${3:-"xenial"}
     dev_mode=${4:-"true"}
     guest_username=${5:-"ubuntu"}
 
-    if [ -z "$datastore_type" ]
-    then
-        echo "You must pass an image type to build, like mysql"
-        exit 1
-    fi
+    exclaim "Building a ${datastore_type} image of trove guest agent for ${guest_os} ${guest_release}."
+
     VALID_SERVICES='mysql percona mariadb redis cassandra couchbase mongodb postgresql couchdb vertica db2 pxc'
     if ! [[ " $VALID_SERVICES " =~ " $datastore_type " ]]; then
-        exclaim "You did not pass in a valid image type. Valid types are:" $VALID_SERVICES
+        exclaim "You did not pass in a valid datastore type. Valid types are:" $VALID_SERVICES
         exit 1
     fi
 

integration/scripts/trovestack

@@ -832,7 +832,6 @@ function cmd_build_image() {
     sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS update
     sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS install qemu git kpartx debootstrap
     sudo -H $HTTP_PROXY pip install diskimage-builder
-    install_devstack_code
 
     exclaim "Use diskimage-builder to actually build the Trove Guest Agent Image."
     build_guest_image $IMAGE_DATASTORE_TYPE $IMAGE_GUEST_OS $IMAGE_GUEST_RELEASE $DEV_MODE

trove/db/sqlalchemy/migrate_repo/versions/019_datastore_fix.py

@@ -16,6 +16,7 @@ from sqlalchemy.schema import MetaData
 from sqlalchemy.sql.expression import insert
 from sqlalchemy.sql.expression import select
 from sqlalchemy.sql.expression import update
+from sqlalchemy import text
 
 from trove.common import cfg
 from trove.db.sqlalchemy.migrate_repo.schema import Table
@@ -53,9 +54,9 @@ def create_legacy_version(datastores_table,
 def find_image(service_name):
     image_table = Table('service_images', meta, autoload=True)
     image = select(
-        columns=["id", "image_id", "service_name"],
+        columns=[text("id"), text("image_id"), text("service_name")],
         from_obj=image_table,
-        whereclause="service_name='%s'" % service_name,
+        whereclause=text("service_name='%s'" % service_name),
         limit=1
     ).execute().fetchone()
 
@@ -66,9 +67,9 @@ def find_image(service_name):
 
 def has_instances_wo_datastore_version(instances_table):
     instance = select(
-        columns=["id"],
+        columns=[text("id")],
         from_obj=instances_table,
-        whereclause="datastore_version_id is NULL",
+        whereclause=text("datastore_version_id is NULL"),
         limit=1
     ).execute().fetchone()
 
@@ -77,9 +78,9 @@ def has_instances_wo_datastore_version(instances_table):
 
 def find_all_instances_wo_datastore_version(instances_table):
     instances = select(
-        columns=["id"],
+        columns=[text("id")],
         from_obj=instances_table,
-        whereclause="datastore_version_id is NULL"
+        whereclause=text("datastore_version_id is NULL")
     ).execute()
 
     return instances
@@ -107,16 +108,16 @@ def upgrade(migrate_engine):
         for instance in instances:
             update(
                 table=instance_table,
-                whereclause="id='%s'" % instance.id,
+                whereclause=text("id='%s'" % instance.id),
                 values=dict(datastore_version_id=version_id)
             ).execute()
 
     constraint_names = db_utils.get_foreign_key_constraint_names(
         engine=migrate_engine,
         table='instances',
-        columns=['datastore_version_id'],
+        columns=[text('datastore_version_id')],
         ref_table='datastore_versions',
-        ref_columns=['id'])
+        ref_columns=[text('id')])
     db_utils.drop_foreign_key_constraints(
         constraint_names=constraint_names,
         columns=[instance_table.c.datastore_version_id],

trove/db/sqlalchemy/migrate_repo/versions/040_module_priority.py

@@ -17,6 +17,7 @@
 from sqlalchemy.schema import Column
 from sqlalchemy.schema import MetaData
 from sqlalchemy.sql.expression import update
+from sqlalchemy import text
 
 from trove.db.sqlalchemy.migrate_repo.schema import Boolean
 from trove.db.sqlalchemy.migrate_repo.schema import Integer
@@ -44,5 +45,5 @@ def upgrade(migrate_engine):
     # mark all non-visible, auto-apply and all-tenant modules as is_admin
     update(table=modules,
            values=dict(is_admin=1),
-           whereclause="visible=0 or auto_apply=1 or tenant_id is null"
+           whereclause=text("visible=0 or auto_apply=1 or tenant_id is null")
            ).execute()