Remove MySQL

Even though there is nothing to read from secure.conf anymore, it
is kept around intact since we may want to use this for ZooKeeper
credentials at some point.

Change-Id: Ieb3a93b09c889f74da3463494957335aaaa9f40f

README.rst

@@ -47,29 +47,6 @@ If the cloud being used has no default_floating_pool defined in nova.conf,
 you will need to define a pool name using the nodepool yaml file to use
 floating ips.
 
-
-Set up database for interactive testing:
-
-.. code-block:: bash
-
-    mysql -u root
-
-    mysql> create database nodepool;
-    mysql> GRANT ALL ON nodepool.* TO 'nodepool'@'localhost';
-    mysql> flush privileges;
-
-Set up database for unit tests:
-
-.. code-block:: bash
-
-    mysql -u root
-    mysql> grant all privileges on *.* to 'openstack_citest'@'localhost' identified by 'openstack_citest' with grant option;
-    mysql> flush privileges;
-    mysql> create database openstack_citest;
-
-Note that the script tools/test-setup.sh can be used for the step
-above.
-
 Export variable for your ssh key so you can log into the created instances:
 
 .. code-block:: bash
@@ -92,9 +69,3 @@ Use the following tool to check on progress:
 .. code-block:: bash
 
     nodepool image-list
-
-After each run (the fake nova provider is only in-memory):
-
-.. code-block:: bash
-
-    mysql> delete from snapshot_image; delete from node;

bindep.txt

@@ -1,8 +1,6 @@
 # This is a cross-platform list tracking distribution packages needed by tests;
 # see http://docs.openstack.org/infra/bindep/ for additional information.
 
-mysql-client [test]
-mysql-server [test]
 python-dev [platform:dpkg test]
 python-devel [platform:rpm test]
 zookeeperd [platform:dpkg test]

devstack/plugin.sh

@@ -101,7 +101,6 @@ EOF
 function nodepool_write_config {
     sudo mkdir -p $(dirname $NODEPOOL_CONFIG)
     sudo mkdir -p $(dirname $NODEPOOL_SECURE)
-    local dburi=$(database_connection_url nodepool)
 
     cat > /tmp/logging.conf <<EOF
 [formatters]
@@ -149,12 +148,7 @@ EOF
     sudo mv /tmp/logging.conf $NODEPOOL_LOGGING
 
     cat > /tmp/secure.conf << EOF
-[database]
+# Empty
-# The mysql password here may be different depending on your
-# devstack install, you should double check it (the devstack var
-# is MYSQL_PASSWORD and if unset devstack should prompt you for
-# the value).
-dburi: $dburi
 EOF
     sudo mv /tmp/secure.conf $NODEPOOL_SECURE
 
@@ -174,11 +168,6 @@ EOF
 # example element.
 elements-dir: $(dirname $NODEPOOL_CONFIG)/elements
 images-dir: $NODEPOOL_DIB_BASE_PATH/images
-# The mysql password here may be different depending on your
-# devstack install, you should double check it (the devstack var
-# is MYSQL_PASSWORD and if unset devstack should prompt you for
-# the value).
-dburi: '$dburi'
 
 zookeeper-servers:
   - host: localhost
@@ -378,7 +367,6 @@ EOF
     mkdir -p $HOME/.cache/openstack/
 }
 
-# Initialize database
 # Create configs
 # Setup custom flavor
 function configure_nodepool {
@@ -390,10 +378,6 @@ function configure_nodepool {
 
     # write the elements
     nodepool_write_elements
-
-    # builds a fresh db
-    recreate_database nodepool
-
 }
 
 function start_nodepool {

doc/source/configuration.rst

@@ -4,23 +4,8 @@ Configuration
 =============
 
 Nodepool reads its secure configuration from ``/etc/nodepool/secure.conf``
-by default. The secure file is a standard ini config file, with
+by default. The secure file is a standard ini config file. Note that this
-one section for the database.
+file is currently unused, but may be in the future.
-
-  [database]
-  dburi={dburi}
-
-Following settings are available::
-
-**required**
-
-  ``dburi``
-  Indicates the URI for the database connection.  See the `SQLAlchemy
-  documentation
-  <http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls>`_
-  for the syntax.  Example::
-
-    dburi='mysql+pymysql://nodepool@localhost/nodepool'
 
 Nodepool reads its configuration from ``/etc/nodepool/nodepool.yaml``
 by default.  The configuration file follows the standard YAML syntax

doc/source/installation.rst

@@ -55,8 +55,7 @@ Configuration
 Nodepool has two required configuration files: secure.conf and
 nodepool.yaml, and an optional logging configuration file logging.conf.
 The secure.conf file is used to store nodepool configurations that contain
-sensitive data, such as the Nodepool database password and Jenkins
+sensitive data. The nodepool.yaml files is used to store all other
-api key. The nodepool.yaml files is used to store all other
 configurations.
 
 The logging configuration file is in the standard python logging

nodepool/cmd/config_validator.py

@@ -105,7 +105,6 @@ class ConfigValidator:
         top_level = {
             'elements-dir': str,
             'images-dir': str,
-            'dburi': str,
             'zookeeper-servers': [{
                 'host': str,
                 'port': int,

nodepool/config.py

@@ -131,12 +131,10 @@ def loadConfig(config_path):
 
     newconfig = Config()
     newconfig.db = None
-    newconfig.dburi = None
     newconfig.providers = {}
     newconfig.labels = {}
     newconfig.elementsdir = config.get('elements-dir')
     newconfig.imagesdir = config.get('images-dir')
-    newconfig.dburi = None
     newconfig.provider_managers = {}
     newconfig.zookeeper_servers = {}
     newconfig.diskimages = {}
@@ -277,7 +275,7 @@ def loadSecureConfig(config, secure_config_path):
     secure = ConfigParser.ConfigParser()
     secure.readfp(open(secure_config_path))
 
-    config.dburi = secure.get('database', 'dburi')
+    #config.dburi = secure.get('database', 'dburi')
 
 
 def _cloudKwargsFromProvider(provider):

nodepool/nodedb.py

@@ -1,247 +0,0 @@
-# Copyright (C) 2011-2014 OpenStack Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-#
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import time
-
-# States:
-# The cloud provider is building this machine.  We have an ID, but it's
-# not ready for use.
-BUILDING = 1
-# The machine is ready for use.
-READY = 2
-# This can mean in-use, or used but complete.
-USED = 3
-# Delete this machine immediately.
-DELETE = 4
-# Keep this machine indefinitely.
-HOLD = 5
-# Acceptance testing (pre-ready)
-TEST = 6
-
-
-STATE_NAMES = {
-    BUILDING: 'building',
-    READY: 'ready',
-    USED: 'used',
-    DELETE: 'delete',
-    HOLD: 'hold',
-    TEST: 'test',
-    }
-
-from sqlalchemy import Table, Column, Integer, String, \
-    MetaData, create_engine
-from sqlalchemy.orm import scoped_session, mapper
-from sqlalchemy.orm.session import Session, sessionmaker
-
-metadata = MetaData()
-
-node_table = Table(
-    'node', metadata,
-    Column('id', Integer, primary_key=True),
-    Column('provider_name', String(255), index=True, nullable=False),
-    Column('label_name', String(255), index=True, nullable=False),
-    Column('target_name', String(255), index=True, nullable=False),
-    Column('manager_name', String(255)),
-    # Machine name
-    Column('hostname', String(255), index=True),
-    # Eg, jenkins node name
-    Column('nodename', String(255), index=True),
-    # Provider assigned id for this machine
-    Column('external_id', String(255)),
-    # Provider availability zone for this machine
-    Column('az', String(255)),
-    # Primary IP address
-    Column('ip', String(255)),
-    # Internal/fixed IP address
-    Column('ip_private', String(255)),
-    # One of the above values
-    Column('state', Integer),
-    # Time of last state change
-    Column('state_time', Integer),
-    # Comment about the state of the node - used to annotate held nodes
-    Column('comment', String(255)),
-    mysql_engine='InnoDB',
-    )
-job_table = Table(
-    'job', metadata,
-    Column('id', Integer, primary_key=True),
-    # The name of the job
-    Column('name', String(255), index=True),
-    # Automatically hold up to this number of nodes that fail this job
-    Column('hold_on_failure', Integer),
-    mysql_engine='InnoDB',
-    )
-
-
-class Node(object):
-    def __init__(self, provider_name, label_name, target_name, az,
-                 hostname=None, external_id=None, ip=None, ip_private=None,
-                 manager_name=None, state=BUILDING, comment=None):
-        self.provider_name = provider_name
-        self.label_name = label_name
-        self.target_name = target_name
-        self.manager_name = manager_name
-        self.external_id = external_id
-        self.az = az
-        self.ip = ip
-        self.ip_private = ip_private
-        self.hostname = hostname
-        self.state = state
-        self.comment = comment
-
-    def delete(self):
-        session = Session.object_session(self)
-        session.delete(self)
-        session.commit()
-
-    @property
-    def state(self):
-        return self._state
-
-    @state.setter
-    def state(self, state):
-        self._state = state
-        self.state_time = int(time.time())
-        session = Session.object_session(self)
-        if session:
-            session.commit()
-
-
-class Job(object):
-    def __init__(self, name=None, hold_on_failure=0):
-        self.name = name
-        self.hold_on_failure = hold_on_failure
-
-    def delete(self):
-        session = Session.object_session(self)
-        session.delete(self)
-        session.commit()
-
-
-mapper(Job, job_table)
-
-
-mapper(Node, node_table,
-       properties=dict(
-           _state=node_table.c.state))
-
-
-class NodeDatabase(object):
-    def __init__(self, dburi):
-        engine_kwargs = dict(echo=False, pool_recycle=3600)
-        if 'sqlite:' not in dburi:
-            engine_kwargs['max_overflow'] = -1
-
-        self.engine = create_engine(dburi, **engine_kwargs)
-        metadata.create_all(self.engine)
-        self.session_factory = sessionmaker(bind=self.engine)
-        self.session = scoped_session(self.session_factory)
-
-    def getSession(self):
-        return NodeDatabaseSession(self.session)
-
-
-class NodeDatabaseSession(object):
-    def __init__(self, session):
-        self.session = session
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, etype, value, tb):
-        if etype:
-            self.session().rollback()
-        else:
-            self.session().commit()
-        self.session().close()
-        self.session = None
-
-    def abort(self):
-        self.session().rollback()
-
-    def commit(self):
-        self.session().commit()
-
-    def delete(self, obj):
-        self.session().delete(obj)
-
-    def getNodes(self, provider_name=None, label_name=None, target_name=None,
-                 state=None):
-        exp = self.session().query(Node).order_by(
-            node_table.c.provider_name,
-            node_table.c.label_name)
-        if provider_name:
-            exp = exp.filter_by(provider_name=provider_name)
-        if label_name:
-            exp = exp.filter_by(label_name=label_name)
-        if target_name:
-            exp = exp.filter_by(target_name=target_name)
-        if state:
-            exp = exp.filter(node_table.c.state == state)
-        return exp.all()
-
-    def createNode(self, *args, **kwargs):
-        new = Node(*args, **kwargs)
-        self.session().add(new)
-        self.commit()
-        return new
-
-    def getNode(self, id):
-        nodes = self.session().query(Node).filter_by(id=id).all()
-        if not nodes:
-            return None
-        return nodes[0]
-
-    def getNodeByHostname(self, hostname):
-        nodes = self.session().query(Node).filter_by(hostname=hostname).all()
-        if not nodes:
-            return None
-        return nodes[0]
-
-    def getNodeByNodename(self, nodename):
-        nodes = self.session().query(Node).filter_by(nodename=nodename).all()
-        if not nodes:
-            return None
-        return nodes[0]
-
-    def getNodeByExternalID(self, provider_name, external_id):
-        nodes = self.session().query(Node).filter_by(
-            provider_name=provider_name,
-            external_id=external_id).all()
-        if not nodes:
-            return None
-        return nodes[0]
-
-    def getJob(self, id):
-        jobs = self.session().query(Job).filter_by(id=id).all()
-        if not jobs:
-            return None
-        return jobs[0]
-
-    def getJobByName(self, name):
-        jobs = self.session().query(Job).filter_by(name=name).all()
-        if not jobs:
-            return None
-        return jobs[0]
-
-    def getJobs(self):
-        return self.session().query(Job).all()
-
-    def createJob(self, *args, **kwargs):
-        new = Job(*args, **kwargs)
-        self.session().add(new)
-        self.commit()
-        return new

nodepool/tests/__init__.py

@@ -18,17 +18,14 @@
 import glob
 import logging
 import os
-import pymysql
 import random
 import string
 import subprocess
 import threading
 import tempfile
 import time
-import uuid
 
 import fixtures
-import lockfile
 import kazoo.client
 import testtools
 
@@ -230,47 +227,6 @@ class RoundRobinTestCase(object):
                                                          self.allocations))
 
 
-class MySQLSchemaFixture(fixtures.Fixture):
-    def setUp(self):
-        super(MySQLSchemaFixture, self).setUp()
-
-        random_bits = ''.join(random.choice(string.ascii_lowercase +
-                                            string.ascii_uppercase)
-                              for x in range(8))
-        self.name = '%s_%s' % (random_bits, os.getpid())
-        self.passwd = uuid.uuid4().hex
-        lock = lockfile.LockFile('/tmp/nodepool-db-schema-lockfile')
-        with lock:
-            db = pymysql.connect(host="localhost",
-                                 user="openstack_citest",
-                                 passwd="openstack_citest",
-                                 db="openstack_citest")
-            cur = db.cursor()
-            cur.execute("create database %s" % self.name)
-            cur.execute(
-                "grant all on %s.* to '%s'@'localhost' identified by '%s'" %
-                (self.name, self.name, self.passwd))
-            cur.execute("flush privileges")
-
-        self.dburi = 'mysql+pymysql://%s:%s@localhost/%s' % (self.name,
-                                                             self.passwd,
-                                                             self.name)
-        self.addDetail('dburi', testtools.content.text_content(self.dburi))
-        self.addCleanup(self.cleanup)
-
-    def cleanup(self):
-        lock = lockfile.LockFile('/tmp/nodepool-db-schema-lockfile')
-        with lock:
-            db = pymysql.connect(host="localhost",
-                                 user="openstack_citest",
-                                 passwd="openstack_citest",
-                                 db="openstack_citest")
-            cur = db.cursor()
-            cur.execute("drop database %s" % self.name)
-            cur.execute("drop user '%s'@'localhost'" % self.name)
-            cur.execute("flush privileges")
-
-
 class BuilderFixture(fixtures.Fixture):
     def __init__(self, configfile, cleanup_interval):
         super(BuilderFixture, self).__init__()
@@ -296,9 +252,6 @@ class DBTestCase(BaseTestCase):
     def setUp(self):
         super(DBTestCase, self).setUp()
         self.log = logging.getLogger("tests")
-        f = MySQLSchemaFixture()
-        self.useFixture(f)
-        self.dburi = f.dburi
         self.secure_conf = self._setup_secure()
         self.setupZK()
 
@@ -333,7 +286,8 @@ class DBTestCase(BaseTestCase):
         (fd, path) = tempfile.mkstemp()
         with open(configfile) as conf_fd:
             config = conf_fd.read()
-            os.write(fd, config.format(dburi=self.dburi))
+            os.write(fd, config)
+            #os.write(fd, config.format(dburi=self.dburi))
         os.close(fd)
         return path
 

nodepool/tests/fixtures/secure.conf

@@ -1,2 +1 @@
-[database]
+# Empty
-dburi={dburi}

requirements.txt

@@ -6,7 +6,6 @@ python-daemon>=2.0.4,<2.1.0
 extras
 statsd>=3.0
 sqlalchemy>=0.8.2,<1.1.0
-PyMySQL
 PrettyTable>=0.6,<0.8
 # shade has a looser requirement on six than nodepool, so install six first
 six>=1.7.0

tools/fake-secure.conf

@@ -1,2 +1 @@
-[database]
+# Empty
-dburi=mysql+pymysql://nodepool@localhost/nodepool

tools/test-setup.sh

@@ -1,33 +0,0 @@
-#!/bin/bash -xe
-
-# This script will be run by OpenStack CI before unit tests are run,
-# it sets up the test system as needed.
-# Developers should setup their test systems in a similar way.
-
-# This setup needs to be run as a user that can run sudo.
-
-# The root password for the MySQL database; pass it in via
-# MYSQL_ROOT_PW.
-DB_ROOT_PW=${MYSQL_ROOT_PW:-insecure_slave}
-
-# This user and its password are used by the tests, if you change it,
-# your tests might fail.
-DB_USER=openstack_citest
-DB_PW=openstack_citest
-
-sudo -H mysqladmin -u root password $DB_ROOT_PW
-
-# It's best practice to remove anonymous users from the database.  If
-# a anonymous user exists, then it matches first for connections and
-# other connections from that host will not work.
-sudo -H mysql -u root -p$DB_ROOT_PW -h localhost -e "
-    DELETE FROM mysql.user WHERE User='';
-    FLUSH PRIVILEGES;
-    GRANT ALL PRIVILEGES ON *.*
-        TO '$DB_USER'@'%' identified by '$DB_PW' WITH GRANT OPTION;"
-
-# Now create our database.
-mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e "
-    SET default_storage_engine=MYISAM;
-    DROP DATABASE IF EXISTS openstack_citest;
-    CREATE DATABASE openstack_citest CHARACTER SET utf8;"

tools/zuul-nodepool-integration/secure.conf

@@ -1,2 +1 @@
-[database]
+# Empty
-dburi=mysql+pymysql://nodepool@localhost/nodepool