From 85ac496ff6d32ac8693d23ac23f7148c51bb0e3f Mon Sep 17 00:00:00 2001
From: jkilpatr <jkilpatr@redhat.com>
Date: Wed, 23 Aug 2017 10:55:52 -0400
Subject: [PATCH] Cyborg deployment script

This deploys cyborg services across various hosts and sets them up
as boot time services configured to access the local rabbit and mysql
instances and then runs some validations to ensure
that everything started correctly.

This does not have any saftey for multi-controller setups yet, right now
if you hand the playbook 3 hosts under 'controller' it will deploy three
api endpoints and three agents. Which probably won't work right.

Change-Id: I79b6b1ce9bb6766d5e608c89504c4bbaeaba5599
---
 .gitignore                                    |  1 +
 cyborg/agent/__init__.py                      | 19 ------
 cyborg/agent/agent.conf                       |  3 -
 cyborg/agent/agent.py                         | 64 -------------------
 cyborg/agent/manager.py                       | 37 +++++++++++
 cyborg/agent/rpcapi.py                        | 46 +++++++++----
 cyborg/{agent/conf.py => cmd/agent.py}        | 31 +++++----
 cyborg/common/constants.py                    |  1 +
 setup.cfg                                     |  1 +
 setup/deploy-cyborg.yml                       | 22 +++++++
 setup/roles/deploy_agent/tasks/main.yml       | 22 +++++++
 .../openstack-cyborg-agent.service.j2         | 13 ++++
 setup/roles/deploy_api/tasks/main.yml         | 22 +++++++
 .../templates/openstack-cyborg-api.service.j2 | 13 ++++
 setup/roles/deploy_conductor/tasks/main.yml   | 22 +++++++
 .../openstack-cyborg-conductor.service.j2     | 13 ++++
 .../roles/generate_credentials/tasks/main.yml | 20 ++++++
 setup/roles/install_package/tasks/main.yml    | 38 +++++++++++
 setup/roles/template_config/tasks/main.yml    | 13 ++++
 .../template_config/templates/cyborg.conf.j2  |  3 +
 setup/roles/validate_agent/tasks/main.yml     | 13 ++++
 setup/roles/validate_api/tasks/main.yml       | 21 ++++++
 setup/roles/validate_conductor/tasks/main.yml | 13 ++++
 23 files changed, 342 insertions(+), 109 deletions(-)
 delete mode 100644 cyborg/agent/agent.conf
 delete mode 100644 cyborg/agent/agent.py
 create mode 100644 cyborg/agent/manager.py
 rename cyborg/{agent/conf.py => cmd/agent.py} (52%)
 create mode 100644 setup/deploy-cyborg.yml
 create mode 100644 setup/roles/deploy_agent/tasks/main.yml
 create mode 100644 setup/roles/deploy_agent/templates/openstack-cyborg-agent.service.j2
 create mode 100644 setup/roles/deploy_api/tasks/main.yml
 create mode 100644 setup/roles/deploy_api/templates/openstack-cyborg-api.service.j2
 create mode 100644 setup/roles/deploy_conductor/tasks/main.yml
 create mode 100644 setup/roles/deploy_conductor/templates/openstack-cyborg-conductor.service.j2
 create mode 100644 setup/roles/generate_credentials/tasks/main.yml
 create mode 100644 setup/roles/install_package/tasks/main.yml
 create mode 100644 setup/roles/template_config/tasks/main.yml
 create mode 100644 setup/roles/template_config/templates/cyborg.conf.j2
 create mode 100644 setup/roles/validate_agent/tasks/main.yml
 create mode 100644 setup/roles/validate_api/tasks/main.yml
 create mode 100644 setup/roles/validate_conductor/tasks/main.yml

diff --git a/.gitignore b/.gitignore
index 59595b91..4c634dd3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 *.pyc
 *.tox
+*.retry
 *.testrepository
 cyborg.egg-info
diff --git a/cyborg/agent/__init__.py b/cyborg/agent/__init__.py
index 61b32e04..e69de29b 100644
--- a/cyborg/agent/__init__.py
+++ b/cyborg/agent/__init__.py
@@ -1,19 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import pbr.version
-
-
-__version__ = pbr.version.VersionInfo(
-    'cyborg-agent').version_string()
diff --git a/cyborg/agent/agent.conf b/cyborg/agent/agent.conf
deleted file mode 100644
index 2cbfdf3c..00000000
--- a/cyborg/agent/agent.conf
+++ /dev/null
@@ -1,3 +0,0 @@
-[cyborg]
-transport_url=
-server_id=
diff --git a/cyborg/agent/agent.py b/cyborg/agent/agent.py
deleted file mode 100644
index d61a1bec..00000000
--- a/cyborg/agent/agent.py
+++ /dev/null
@@ -1,64 +0,0 @@
-# -*- coding: utf-8 -*-
-
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import conf
-import eventlet
-from oslo_config import cfg
-from oslo_log import log as logging
-import oslo_messaging as messaging
-import rpcapi
-import time
-
-eventlet.monkey_patch()
-
-CONF = cfg.CONF
-
-conf.register_opts(CONF)
-
-LOG = logging.getLogger(__name__)
-logging.register_options(CONF)
-logging.setup(CONF, 'Cyborg.Agent')
-
-CONF(['--config-file', 'agent.conf'])
-
-url = messaging.TransportURL.parse(CONF, url=CONF.cyborg.transport_url)
-transport = messaging.get_notification_transport(CONF, url)
-
-notifier = messaging.Notifier(transport,
-                              driver='messaging',
-                              publisher_id='Cyborg.Agent',
-                              topic='info')
-
-rpc_targets = messaging.Target(topic='cyborg_control',
-                               server=CONF.cyborg.server_id)
-rpc_endpoints = [
-    rpcapi.RPCEndpoint()
-]
-access_policy = messaging.ExplicitRPCAccessPolicy
-rpc_server = messaging.get_rpc_server(transport,
-                                      rpc_targets,
-                                      rpc_endpoints,
-                                      executor='eventlet',
-                                      access_policy=access_policy)
-
-try:
-    print("Cyborg Agent running")
-    rpc_server.start()
-    while True:
-        time.sleep(1)
-except KeyboardInterrupt:
-    print("Stopping server")
-    rpc_server.stop()
-    rpc_server.wait()
diff --git a/cyborg/agent/manager.py b/cyborg/agent/manager.py
new file mode 100644
index 00000000..8fc02a6a
--- /dev/null
+++ b/cyborg/agent/manager.py
@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import oslo_messaging as messaging
+
+from cyborg.conf import CONF
+
+
+class AgentManager(object):
+    """Cyborg Agent manager main class."""
+
+    RPC_API_VERSION = '1.0'
+    target = messaging.Target(version=RPC_API_VERSION)
+
+    def __init__(self, topic, host=None):
+        super(AgentManager, self).__init__()
+        self.topic = topic
+        self.host = host or CONF.host
+
+    def periodic_tasks(self, context, raise_on_error=False):
+        pass
+
+    def hardware_list(self, context, values):
+        """List installed hardware."""
+        pass
diff --git a/cyborg/agent/rpcapi.py b/cyborg/agent/rpcapi.py
index b82136a8..f683dc01 100644
--- a/cyborg/agent/rpcapi.py
+++ b/cyborg/agent/rpcapi.py
@@ -13,18 +13,40 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+"""Client side of the conductor RPC API."""
 
-class RPCEndpoint(object):
+from oslo_config import cfg
+import oslo_messaging as messaging
 
-    # Conductor functions exposed for external calls
-    def __init__(self):
-        pass
-
-    def list_accelerators(self, ctxt):
-        pass
-
-    def update_accelerator(self, ctxt, accelerator):
-        pass
-
-    def discover_accelerators(self, ctxt):
+from cyborg.common import constants
+from cyborg.common import rpc
+from cyborg.objects import base as objects_base
+
+
+CONF = cfg.CONF
+
+
+class AgentAPI(object):
+    """Client side of the Agent RPC API.
+
+    API version history:
+
+    |    1.0 - Initial version.
+
+    """
+
+    RPC_API_VERSION = '1.0'
+
+    def __init__(self, topic=None):
+        super(AgentAPI, self).__init__()
+        self.topic = topic or constants.AGENT_TOPIC
+        target = messaging.Target(topic=self.topic,
+                                  version='1.0')
+        serializer = objects_base.CyborgObjectSerializer()
+        self.client = rpc.get_client(target,
+                                     version_cap=self.RPC_API_VERSION,
+                                     serializer=serializer)
+
+    def hardware_list(self, context, values):
+        """Signal the agent to find local hardware."""
         pass
diff --git a/cyborg/agent/conf.py b/cyborg/cmd/agent.py
similarity index 52%
rename from cyborg/agent/conf.py
rename to cyborg/cmd/agent.py
index 96f5f32e..8663bd13 100644
--- a/cyborg/agent/conf.py
+++ b/cyborg/cmd/agent.py
@@ -11,18 +11,27 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+"""The Cyborg Agent Service."""
+
+import sys
+
 from oslo_config import cfg
-import uuid
+from oslo_service import service
 
-default_opts = [
-    cfg.StrOpt('transport_url',
-               default='',
-               help='Transport url to use for messaging'),
-    cfg.StrOpt('server_id',
-               default=uuid.uuid4(),
-               help='Unique identifier for this agent'),
-]
+from cyborg.common import constants
+from cyborg.common import service as cyborg_service
 
 
-def register_opts(conf):
-    conf.register_opts(default_opts, group='cyborg')
+CONF = cfg.CONF
+
+
+def main():
+    # Parse config file and command line options, then start logging
+    cyborg_service.prepare_service(sys.argv)
+
+    mgr = cyborg_service.RPCService('cyborg.agent.manager',
+                                    'AgentManager',
+                                    constants.AGENT_TOPIC)
+
+    launcher = service.launch(CONF, mgr)
+    launcher.wait()
diff --git a/cyborg/common/constants.py b/cyborg/common/constants.py
index 54e03065..c9c7f985 100644
--- a/cyborg/common/constants.py
+++ b/cyborg/common/constants.py
@@ -15,3 +15,4 @@
 
 
 CONDUCTOR_TOPIC = 'cyborg-conductor'
+AGENT_TOPIC = 'cyborg-agent'
diff --git a/setup.cfg b/setup.cfg
index 5d112f73..9f69da0f 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -28,6 +28,7 @@ console_scripts =
     cyborg-api = cyborg.cmd.api:main
     cyborg-conductor = cyborg.cmd.conductor:main
     cyborg-dbsync = cyborg.cmd.dbsync:main
+    cyborg-agent = cyborg.cmd.agent:main
 
 cyborg.database.migration_backend =
     sqlalchemy = cyborg.db.sqlalchemy.migration
diff --git a/setup/deploy-cyborg.yml b/setup/deploy-cyborg.yml
new file mode 100644
index 00000000..71bde2e6
--- /dev/null
+++ b/setup/deploy-cyborg.yml
@@ -0,0 +1,22 @@
+---
+# This Ansible playbook deploys Cyborg services on an openstack cloud
+
+- hosts: controller
+  remote_user: heat-admin
+  roles:
+    - generate_credentials
+    - install_package
+    - template_config
+    - deploy_api
+    - deploy_conductor
+    - validate_api
+    - validate_conductor
+
+- hosts: compute
+  remote_user: heat-admin
+  roles:
+    - install_package
+    - template_config
+    - deploy_agent
+    - validate_agent
+    #- deploy_drivers
diff --git a/setup/roles/deploy_agent/tasks/main.yml b/setup/roles/deploy_agent/tasks/main.yml
new file mode 100644
index 00000000..6a3899d5
--- /dev/null
+++ b/setup/roles/deploy_agent/tasks/main.yml
@@ -0,0 +1,22 @@
+---
+# Sets up Cyborg api to start at boot
+
+- name: Create Cyborg user
+  user:
+    name: cyborg
+    comment: "cyborg user"
+    createhome: no
+  become: true
+
+- name: Template service file for Cyborg Agent
+  template:
+    src: openstack-cyborg-agent.service.j2
+    dest: /usr/lib/systemd/system/openstack-cyborg-agent.service
+  become: true
+
+- name: Start service and set to run at boot
+  service:
+    name: openstack-cyborg-agent
+    state: started
+    enabled: yes
+  become: true
diff --git a/setup/roles/deploy_agent/templates/openstack-cyborg-agent.service.j2 b/setup/roles/deploy_agent/templates/openstack-cyborg-agent.service.j2
new file mode 100644
index 00000000..676beffa
--- /dev/null
+++ b/setup/roles/deploy_agent/templates/openstack-cyborg-agent.service.j2
@@ -0,0 +1,13 @@
+[Unit]
+Description=OpenStack Accelerator management service
+After=syslog.target network.target
+
+[Service]
+Type=simple
+User=cyborg
+ExecStart=/usr/bin/cyborg-agent
+PrivateTmp=true
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
diff --git a/setup/roles/deploy_api/tasks/main.yml b/setup/roles/deploy_api/tasks/main.yml
new file mode 100644
index 00000000..6c3922a9
--- /dev/null
+++ b/setup/roles/deploy_api/tasks/main.yml
@@ -0,0 +1,22 @@
+---
+# Sets up Cyborg api to start at boot
+
+- name: Create Cyborg user
+  user:
+    name: cyborg
+    comment: "cyborg user"
+    createhome: no
+  become: true
+
+- name: Template service file for Cyborg API
+  template:
+    src: openstack-cyborg-api.service.j2
+    dest: /usr/lib/systemd/system/openstack-cyborg-api.service
+  become: true
+
+- name: Start service and set to run at boot
+  service:
+    name: openstack-cyborg-api
+    state: started
+    enabled: yes
+  become: true
diff --git a/setup/roles/deploy_api/templates/openstack-cyborg-api.service.j2 b/setup/roles/deploy_api/templates/openstack-cyborg-api.service.j2
new file mode 100644
index 00000000..8d40ec84
--- /dev/null
+++ b/setup/roles/deploy_api/templates/openstack-cyborg-api.service.j2
@@ -0,0 +1,13 @@
+[Unit]
+Description=OpenStack Accelerator management service
+After=syslog.target network.target
+
+[Service]
+Type=simple
+User=cyborg
+ExecStart=/usr/bin/cyborg-api
+PrivateTmp=true
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
diff --git a/setup/roles/deploy_conductor/tasks/main.yml b/setup/roles/deploy_conductor/tasks/main.yml
new file mode 100644
index 00000000..d5c41d19
--- /dev/null
+++ b/setup/roles/deploy_conductor/tasks/main.yml
@@ -0,0 +1,22 @@
+---
+# Sets up Cyborg api to start at boot
+
+- name: Create Cyborg user
+  user:
+    name: cyborg
+    comment: "cyborg user"
+    createhome: no
+  become: true
+
+- name: Template service file for Cyborg Conductor
+  template:
+    src: openstack-cyborg-conductor.service.j2
+    dest: /usr/lib/systemd/system/openstack-cyborg-conductor.service
+  become: true
+
+- name: Start service and set to run at boot
+  service:
+    name: openstack-cyborg-conductor
+    state: started
+    enabled: yes
+  become: true
diff --git a/setup/roles/deploy_conductor/templates/openstack-cyborg-conductor.service.j2 b/setup/roles/deploy_conductor/templates/openstack-cyborg-conductor.service.j2
new file mode 100644
index 00000000..10b5ced0
--- /dev/null
+++ b/setup/roles/deploy_conductor/templates/openstack-cyborg-conductor.service.j2
@@ -0,0 +1,13 @@
+[Unit]
+Description=OpenStack Accelerator management service
+After=syslog.target network.target
+
+[Service]
+Type=simple
+User=glance
+ExecStart=/usr/bin/cyborg-conductor
+PrivateTmp=true
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
diff --git a/setup/roles/generate_credentials/tasks/main.yml b/setup/roles/generate_credentials/tasks/main.yml
new file mode 100644
index 00000000..55e8b44e
--- /dev/null
+++ b/setup/roles/generate_credentials/tasks/main.yml
@@ -0,0 +1,20 @@
+---
+
+- name: Create Cyborg mysql user
+  mysql_user:
+    name: cyborg
+    password: "{{ lookup('password', 'credentials/cyborg/mysqlpassword length=15') }}" # Generates a password
+    priv: '*.*:ALL,GRANT' # Do we only need the cyborg database?
+    state: present
+  become: true
+
+- name: Create Cyborg rabbitmq user
+  rabbitmq_user:
+    user: cyborg
+    password: "{{ lookup('password', 'credentials/cyborg/rabbitpassword length=15') }}" # Generates a password
+    vhost: /
+    read_priv: .*
+    write_priv: .*
+    configure_priv: .*
+    state: present
+  become: true
diff --git a/setup/roles/install_package/tasks/main.yml b/setup/roles/install_package/tasks/main.yml
new file mode 100644
index 00000000..561cc1c2
--- /dev/null
+++ b/setup/roles/install_package/tasks/main.yml
@@ -0,0 +1,38 @@
+---
+
+- name: Check if pip is installed
+  shell: "which pip"
+  register: which_pip
+  ignore_errors: true
+
+- name: Install pip
+  package:
+    name: python-pip
+    state: present
+  when: which_pip|failed
+  become: true
+
+- name: Install rsync
+  package:
+    name: rsync
+    state: present
+  become: true
+
+- name: Copy cyborg to host
+  synchronize:
+    src: ../../../cyborg/
+    dest: /tmp/cyborg
+    use_ssh_args: yes
+
+- name: Remove old Cyborg if installed
+  pip:
+    name: cyborg
+    state: absent
+  become: true
+  ignore_errors: true
+
+- name: Install Cyborg using pip
+  pip:
+    name: /tmp/cyborg
+    state: present
+  become: true
diff --git a/setup/roles/template_config/tasks/main.yml b/setup/roles/template_config/tasks/main.yml
new file mode 100644
index 00000000..040f3e24
--- /dev/null
+++ b/setup/roles/template_config/tasks/main.yml
@@ -0,0 +1,13 @@
+---
+
+- name: Create cyborg config dir
+  file:
+    path: /etc/cyborg
+    state: directory
+  become: true
+
+- name: Template Cyborg.conf
+  template:
+    src: cyborg.conf.j2
+    dest: /etc/cyborg/cyborg.conf
+  become: true
diff --git a/setup/roles/template_config/templates/cyborg.conf.j2 b/setup/roles/template_config/templates/cyborg.conf.j2
new file mode 100644
index 00000000..1298560a
--- /dev/null
+++ b/setup/roles/template_config/templates/cyborg.conf.j2
@@ -0,0 +1,3 @@
+[DEFAULT]
+connection=mysql+pymysql://cyborg:{{ lookup('password', 'credentials/cyborg/rabbitpassword length=15') }}@overcloud-controller-0.internalapi/nova?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo
+transport_url=rabbit://cyborg:{{ lookup('password', 'credentials/cyborg/rabbitpassword length=15') }}@overcloud-controller-0.internalapi:5672/?ssl=0
diff --git a/setup/roles/validate_agent/tasks/main.yml b/setup/roles/validate_agent/tasks/main.yml
new file mode 100644
index 00000000..8d86faa3
--- /dev/null
+++ b/setup/roles/validate_agent/tasks/main.yml
@@ -0,0 +1,13 @@
+---
+
+- name: Check Agent status
+  service:
+    name: openstack-cyborg-agent
+    state: started
+    enabled: yes
+  become: true
+  register: result
+
+- name: Fail if Agent is not up
+  fail: msg="Cyborg Agent did not start correctly!"
+  when: result.status.ActiveState == "failed"
diff --git a/setup/roles/validate_api/tasks/main.yml b/setup/roles/validate_api/tasks/main.yml
new file mode 100644
index 00000000..c3188ede
--- /dev/null
+++ b/setup/roles/validate_api/tasks/main.yml
@@ -0,0 +1,21 @@
+---
+
+- name: Check API status
+  service:
+    name: openstack-cyborg-api
+    state: started
+    enabled: yes
+  become: true
+  register: result
+
+- name: Fail if API did not start
+  fail: msg="Cyborg API did not start correctly!"
+  when: result.status.ActiveState == "failed"
+
+- name: Make a request to the cyborg API endpoint
+  wait_for:
+    host: localhost
+    port: 6666
+    state: started
+    delay: 1
+    timeout: 60
diff --git a/setup/roles/validate_conductor/tasks/main.yml b/setup/roles/validate_conductor/tasks/main.yml
new file mode 100644
index 00000000..b36008fa
--- /dev/null
+++ b/setup/roles/validate_conductor/tasks/main.yml
@@ -0,0 +1,13 @@
+---
+
+- name: Check if Conductor is running
+  service:
+    name: openstack-cyborg-conductor
+    state: started
+    enabled: yes
+  become: true
+  register: result
+
+- name: Fail if Conductor is not running
+  fail: msg="Cyborg Conductor did not start correctly!"
+  when: result.status.ActiveState == "failed"