Add kolide cluster tests

Change-Id: I4452b5f42eb9c09da48e657d20144f09fe785878
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

osquery/installOSquery.yml

@@ -26,16 +26,12 @@
   gather_facts: "{{ osa_gather_facts | default(True) }}"
 
   tasks:
-    - name: Fetch Certificates
-      fetch:
-        flat: yes
-        src: "{{ item.src }}"
-        dest: "{{ item.dest }}"
+    - name: create osquery dir
+      file:
+        path: "{{ item }}"
+        state: directory
       with_items:
-        - src: "{{ kolide_fleet_ssl_cert }}"
-          dest: "/tmp/{{ kolide_fleet_ssl_cert | basename }}"
-        - src: "{{ kolide_fleet_ssl_key }}"
-          dest: "/tmp/{{ kolide_fleet_ssl_key | basename }}"
+        - /etc/osquery
 
     - name: Retrieve Enrollment Token
       command: /usr/local/bin/fleetctl get enroll-secret
@@ -48,10 +44,24 @@
 
     - name: Write enroll secret
       copy:
-        dest: "/tmp/{{ osquery_enroll_secret_file | basename }}"
+        dest: "{{ osquery_enroll_secret_file }}"
         content: "{{ kolide_fleet_enroll_secret }}"
         mode: "0640"
 
+    - name: Fetch Certificates
+      fetch:
+        flat: yes
+        src: "{{ item.src }}"
+        dest: "{{ item.dest }}"
+        mode: "0600"
+      with_items:
+        - src: "{{ kolide_fleet_ssl_cert }}"
+          dest: "/tmp/{{ kolide_fleet_ssl_cert | basename }}"
+        - src: "{{ kolide_fleet_ssl_key }}"
+          dest: "/tmp/{{ kolide_fleet_ssl_key | basename }}"
+        - src: "{{ osquery_enroll_secret_file }}"
+          dest: "/tmp/{{ osquery_enroll_secret_file | basename }}"
+
 
 - name: Install osquery
   hosts: "hosts:all_containers"
@@ -76,6 +86,7 @@
       copy:
         src: "{{ item.src }}"
         dest: "{{ item.dest }}"
+        mode: "0640"
       with_items:
         - dest: "{{ kolide_fleet_ssl_cert }}"
           src: "/tmp/{{ kolide_fleet_ssl_cert | basename }}"

osquery/roles/fleet/tasks/createFleetDB.yml

@@ -22,6 +22,7 @@
     state: "present"
   delegate_to: "{{ groups['mariadb_all'][0] }}"
   no_log: False
+  run_once: true
   tags:
     - fleet_db_install
 
@@ -39,5 +40,6 @@
   delegate_to: "{{ groups['mariadb_all'][0] }}"
   with_items: "{{ grant_list | default(['localhost', '%']) }}"
   no_log: False
+  run_once: true
   tags:
     - fleet_db_install

osquery/roles/fleet/tasks/fleetConfig.yml

@@ -1,28 +0,0 @@
----
-# Copyright 2016, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: create fleet dir
-  file:
-    path: /etc/fleet
-    state: directory
-
-- name: Drop fleet conf file
-  template:
-    src: templates/fleet_config.yml.j2
-    dest: /etc/fleet/fleet_config.yml
-  notify:
-    - Restart kolide (systemd)
-  tags:
-    - fleet_config

osquery/roles/fleet/tasks/fleetGetEnrollmentToken.yml

@@ -1,23 +0,0 @@
----
-# Copyright 2016, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: retrieve Enrollment Token
-  command: /usr/local/bin/fleetctl get enroll-secret
-  changed_when: false
-  register: _enrollment_token
-
-- name: Set kolide fleet enrollment token fact
-  set_fact:
-    kolide_fleet_enroll_secret: "{{ _enrollment_token.stdout }}"

osquery/roles/fleet/tasks/fleetRegisterAdmin.yml

@@ -14,17 +14,17 @@
 # limitations under the License.
 
 - name: set fleetctl default context
-  command: /usr/local/bin/fleetctl config set --address https://localhost:{{ kolide_fleet_port }} --tls-skip-verify
+  command: /usr/local/bin/fleetctl config set --address https://127.0.0.1:{{ kolide_fleet_port }} --tls-skip-verify
   changed_when: false
 
 - name: register admin account
   command: /usr/local/bin/fleetctl setup --email {{ kolide_fleet_admin_email }} --password {{ kolide_fleet_admin_password }}
-  changed_when: false
-  failed_when: false
   register: fleet_register_admin
+  changed_when:
+    - fleet_register_admin.rc == 0
+  failed_when:
+    - fleet_register_admin.rc not in [0, 1]
 
 - name: login admin account
   command: /usr/local/bin/fleetctl login --email {{ kolide_fleet_admin_email }} --password {{ kolide_fleet_admin_password }}
   changed_when: false
-  when:
-    - fleet_register_admin['stderr'] == "Kolide Fleet has already been setup"

osquery/roles/fleet/tasks/fleetRequirements.yml

@@ -1,29 +0,0 @@
----
-# Copyright 2016, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Run the systemd service role
-  include_role:
-    name: redis
-    private: true
-
-- name: create fleet dir
-  file:
-    path: /etc/fleet/ssl
-    state: directory
-
-- name: Drop fleet conf file
-  template:
-    src: templates/fleet_config.yml.j2
-    dest: /etc/fleet/fleet_config.yml

osquery/roles/fleet/tasks/fleetSSL.yml

@@ -13,15 +13,43 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Ensure the private ssl directory exists
-  file:
-    dest: "/etc/ssl/private"
-    state: "directory"
+- include_tasks: fleetSSLkeyCreate.yml
+  when:
+    - kolide_fleet_user_ssl_cert is not defined or
+      kolide_fleet_user_ssl_key is not defined
+
+- name: Drop user provided ssl cert
+  copy:
+    src: "{{ kolide_fleet_user_ssl_cert }}"
+    dest: "{{ kolide_fleet_ssl_cert }}"
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  when:
+    - kolide_fleet_user_ssl_cert is defined
   tags:
     - fleet-ssl
 
-- include_tasks: fleetSSLkeyCreate.yml
+- name: Drop user provided ssl key
+  copy:
+    src: "{{ kolide_fleet_user_ssl_key }}"
+    dest: "{{ kolide_fleet_ssl_key }}"
+    owner: "root"
+    group: "root"
+    mode: "0640"
   when:
-    - kolide_fleet_user_ssl_cert is not defined or kolide_fleet_user_ssl_key is not defined
+    - kolide_fleet_user_ssl_key is defined
+  tags:
+    - fleet-ssl
 
-- include_tasks: fleetSSLuserProvided.yml
+- name: Drop user provided ssl CA cert
+  copy:
+    src: "{{ kolide_fleet_user_ssl_ca_cert }}"
+    dest: "{{ kolide_fleet_ssl_ca_cert }}"
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  when:
+    - kolide_fleet_user_ssl_ca_cert is defined
+  tags:
+    - fleet-ssl

osquery/roles/fleet/tasks/fleetSSLkeyCreate.yml

@@ -52,9 +52,9 @@
         issuer:
           O: OpenStack-Ansible-OPS
 
-    - name: Synchronize certifactes back
-      synchronize:
-        mode: pull
+    - name: Fetch Certificates
+      fetch:
+        flat: yes
         src: "{{ item.src }}"
         dest: "{{ item.dest }}"
       with_items:

osquery/roles/fleet/tasks/fleetSSLuserProvided.yml

@@ -1,47 +0,0 @@
----
-# Copyright 2015, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Drop user provided ssl cert
-  copy:
-    src: "{{ kolide_fleet_user_ssl_cert }}"
-    dest: "{{ kolide_fleet_ssl_cert }}"
-    owner: "root"
-    group: "root"
-    mode: "0644"
-  when: kolide_fleet_user_ssl_cert is defined
-  tags:
-    - fleet-ssl
-
-- name: Drop user provided ssl key
-  copy:
-    src: "{{ kolide_fleet_user_ssl_key }}"
-    dest: "{{ kolide_fleet_ssl_key }}"
-    owner: "root"
-    group: "root"
-    mode: "0640"
-  when: kolide_fleet_user_ssl_key is defined
-  tags:
-    - fleet-ssl
-
-- name: Drop user provided ssl CA cert
-  copy:
-    src: "{{ kolide_fleet_user_ssl_ca_cert }}"
-    dest: "{{ kolide_fleet_ssl_ca_cert }}"
-    owner: "root"
-    group: "root"
-    mode: "0644"
-  when: kolide_fleet_user_ssl_ca_cert is defined
-  tags:
-    - fleet-ssl

osquery/roles/fleet/tasks/fleetServerInstall.yml

@@ -13,6 +13,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- name: create fleet dir
+  file:
+    path: "{{ item }}"
+    state: directory
+  with_items:
+    - /etc/fleet
+    - /etc/fleet/ssl
+    - /etc/ssl/private
+
+- name: Drop fleet conf file
+  template:
+    src: templates/fleet_config.yml.j2
+    dest: /etc/fleet/fleet_config.yml
+  notify:
+    - Restart kolide (systemd)
+  tags:
+    - fleet_config
+
 - name: Ensure required disto packages are installed
   package:
     name: "{{ kolide_fleet_distro_packages }}"

osquery/roles/fleet/tasks/main.yml

@@ -25,21 +25,20 @@
   tags:
     - always
 
+- name: Run the systemd service role
+  include_role:
+    name: redis
+    private: true
+
 # install kolide fleet server
 - include_tasks: fleetServerInstall.yml
 
-# install software requirements
-- include_tasks: fleetRequirements.yml
-
 # install SSL certs
 - include_tasks: createFleetDB.yml
 
 # install SSL certs
 - include_tasks: fleetSSL.yml
 
-# drop the configuration
-- include_tasks: fleetConfig.yml
-
 # add files for systemd
 - include_tasks:  fleetService.yml
 
@@ -51,6 +50,3 @@
 
 # configure kolide fleet & set admin account
 - include_tasks:  fleetRegisterAdmin.yml
-
-# retrieve and set enrollment token
-- include_tasks:  fleetGetEnrollmentToken.yml

osquery/tests/inventory/test-container-inventory.yml

@@ -7,10 +7,18 @@ all:
       ansible_host: 127.0.0.1
       ansible_user: root
 
-    kolide-fleet1:
+    kolide-fleet0:
       ansible_host: 172.29.236.100
       ansible_user: root
 
+    kolide-fleet1:
+      ansible_host: 172.29.236.101
+      ansible_user: root
+
+    kolide-fleet2:
+      ansible_host: 172.29.236.102
+      ansible_user: root
+
 
 hosts:
   vars:
@@ -41,7 +49,9 @@ all_containers:
       children:
         mariadb:
           hosts:
+            kolide-fleet0: {}
             kolide-fleet1: {}
+            kolide-fleet2: {}
 
     fleet_all:
       children:
@@ -49,4 +59,6 @@ all_containers:
           children:
             kolide-fleet:
               hosts:
+                kolide-fleet0: {}
                 kolide-fleet1: {}
+                kolide-fleet2: {}

zuul.d/jobs.yaml

@@ -94,6 +94,14 @@
     files:
       - ^osquery/.*
 
+- job:
+    name: "openstack-ansible-ops:osquery-ubuntu-xenial-clustered"
+    parent: "openstack-ansible-ops:osquery-ubuntu-xenial"
+    nodeset: ubuntu-bionic
+    vars:
+      osa_test_repo: "openstack/openstack-ansible-ops"
+      test_clustered_kolide: true
+
 - job:
     name: "openstack-ansible-ops:osquery-ubuntu-bionic"
     parent: "openstack-ansible-ops:osquery-ubuntu-xenial"
@@ -103,7 +111,6 @@
     name: "openstack-ansible-ops:osquery-ubuntu-bionic-clustered"
     parent: "openstack-ansible-ops:osquery-ubuntu-xenial"
     nodeset: ubuntu-bionic
-    voting: true
     vars:
       osa_test_repo: "openstack/openstack-ansible-ops"
       test_clustered_kolide: true

zuul.d/project.yaml

@@ -30,7 +30,9 @@
         - openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial-clustered
         - openstack-ansible-ops:elk_metrics_6x-ubuntu-bionic-clustered
         - openstack-ansible-ops:osquery-ubuntu-xenial
+        - openstack-ansible-ops:osquery-ubuntu-xenial-clustered
         - openstack-ansible-ops:osquery-ubuntu-bionic
+        - openstack-ansible-ops:osquery-ubuntu-bionic-clustered
     gate:
       jobs:
         - openstack-ansible-ops:elk_metrics_6x-ubuntu-bionic