From a5ca1d5837fc118ccd99e3ce38a5d5306355342c Mon Sep 17 00:00:00 2001 From: Daniel Pawlik Date: Tue, 8 Mar 2022 16:28:19 +0100 Subject: [PATCH] Add Opensearch configuration information; change README file; enable md Change-Id: Ic2b64212cfec88bc705ae1f9ae8c5118d9b641c8 --- README.rst | 123 +++++++++++- doc/requirements.txt | 1 + doc/source/conf.py | 4 +- doc/source/index.rst | 18 +- doc/source/infra-changelog.md | 61 ++++++ .../{ansible-role.rst => logscraper-role.rst} | 0 doc/source/logsender-role.rst | 1 + doc/source/opensearch-configuration.md | 1 + opensearch-config/README.md | 184 ++++++++++++++---- setup.cfg | 4 +- 10 files changed, 347 insertions(+), 50 deletions(-) create mode 100644 doc/source/infra-changelog.md rename doc/source/{ansible-role.rst => logscraper-role.rst} (100%) create mode 120000 doc/source/logsender-role.rst create mode 120000 doc/source/opensearch-configuration.md diff --git a/README.rst b/README.rst index e41990f..71383da 100644 --- a/README.rst +++ b/README.rst @@ -5,17 +5,16 @@ The goal of this repository is to provide and check functionality of new log processing system base on zuul log scraper tool. -Zuul Log Scraper ----------------- +Log Scraper +----------- -The Zuul Log Scraper tool is responsible for periodical +The Log Scraper tool is responsible for periodical check by using Zuul CI API if there are new builds available and if there are some, it would push the informations to the log processing system. - -Zuul Log Sender ---------------- +Log Sender +---------- The Zuul Log Sender tool is responsible for periodical check directory, if there are some files that should be send to the @@ -24,6 +23,118 @@ NOTE: build directories that does not provide files `buildinfo` and `inventory.yaml` file are skipped. +Available workflows +------------------- + +The Openstack CI Log Processing project is providing two configurations +for sending logs from Zuul CI to the Opensearch host. + +1. Logscraper, log gearman client, log gearman worker, logstash + +With this solution, log workflow looks like: + +.. code-block:: shell + + +------------------+ 1. Get last builds info +----------------+ + | |-------------------------> | | + | Logscraper | | Zuul API | + | |<------------------------- | | + +------------------+ 2. Fetch data +----------------+ + | + | + +-------------------+ + | + | + 3. Send queue | + logs to gearman | + client | + v + +------------------+ + | | + | Log gearman | + | client | + +------------------+ + +-------------- --------------+ + | | | + | 4. Consume queue, | | + | download log files | | + | | | + v v v + +---------------+ +----------------+ +---------------+ + | Log gearman | | Log gearman | | Log gearman | + | worker | | worker | | worker | + +---------------+ +----------------+ +---------------+ + | | | + | 5. Send to | | + | Logstash | | + | v | + | +----------------+ | + | | | | + +---------> | Logstash | <-------+ + | | + +----------------+ + | + 6. Send to | + Opensearch | + | + +--------v--------+ + | | + | Opensearch | + | | + +-----------------+ + + +On the beginning, this project was designed to use that solution, but +it have a few bottlenecks: +- log gearman client can use many memory, when log gearman worker is not fast, +- one log gearman worker is not enough even on small infrastructure, +- logstash service can fail, +- logscraper is checking if log files are available, then log gearman + is downloading the logs, which can make an issue on log sever, that + host does not have free socket. + +You can deploy your log workflow by using example Ansible playbook that +you can find in `ansible/playbooks/check-services.yml` in this project. + +2. Logscraper, logsender + +This workflow removes bottlenecks by removing: log gearman client, +log gearman worker and logstash service. Logs are downloaded when +available to the disk, then parsed by logsender and send directly to the +Opensearch host. + +With this solution, log workflow looks like: + +.. code-block:: shell + + 1. Get last + +----------------+ builds info +-----------------+ + | | --------------------> | | + | Logscraper | | Zuul API | + | | <-------------------- | | + +----------------+ 2. Fetch data +-----------------+ + | + 3. Download logs; | + include inventory.yaml | + and build info v + +----------------+ + | | + | Logsender | + | | + +----------------+ + | + 4. Parse log files; | + add required fields; | + send to Opensearch v + +-----------------+ + | | + | Opensearch | + | | + +-----------------+ + +You can deploy your log workflow by using example Ansible playbook that +you can find in `ansible/playbooks/check-services-sender.yml` in this project. + Testing ------- diff --git a/doc/requirements.txt b/doc/requirements.txt index 4a62566..782153b 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,3 +1,4 @@ sphinx!=1.6.6,!=1.6.7 # BSD openstackdocstheme>=1.32.0 # Apache-2.0 zuul-sphinx>=0.1.1 +recommonmark>=0.5.0 diff --git a/doc/source/conf.py b/doc/source/conf.py index 191c733..9456e9c 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -12,14 +12,14 @@ sys.path.insert(0, os.path.abspath('.')) # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['zuul_sphinx'] +extensions = ['openstackdocstheme', 'recommonmark'] # We have roles split between zuul-suitable roles at top level roles/* # (automatically detected by zuul-sphinx) and playbook-specific roles # (might have plugins, etc that make them unsuitable as potential zuul # roles). Document both. zuul_role_paths = ['playbooks/roles'] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ['.rst', '.md'] # The master toctree document. master_doc = 'index' # General information about the project. diff --git a/doc/source/index.rst b/doc/source/index.rst index 8237840..8665850 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -13,24 +13,28 @@ service-incident@lists.opendev.org. Contents: +Indices and tables +================== + +* :ref:`search` + .. sidebar:: HOWTOs * :doc:`logscraper` + * :doc:`logscraper-role` * :doc:`logsender` - * :doc:`ansible-role` + * :doc:`logsender-role` * :doc:`loggearman` * :doc:`loggearman-role` .. toctree:: :maxdepth: 2 + infra-changelog + opensearch-configuration logscraper + logscraper-role logsender - ansible-role + logsender-role loggearman loggearman-role - -Indices and tables -================== - -* :ref:`search` diff --git a/doc/source/infra-changelog.md b/doc/source/infra-changelog.md new file mode 100644 index 0000000..c72af18 --- /dev/null +++ b/doc/source/infra-changelog.md @@ -0,0 +1,61 @@ +# Infrastructure changelog + +## Changelog + +#### 13.03.2022 + +What has been done: + +- recreated Opensearch cluster + - resize storage from 50GB to 1TB +- changed cloudformation stack name from opensearchteststack to opensearchstack, +- added security rule that allows logscraper01.openstack.org to push logs to the Logstash service, +- renamed Logstash container service deployment from pre-prod-Cluster to production-Cluster. + +Executed commands: + +``` shell +# Delete stacks if they exist +echo "Deleting Logstash stack..." +aws cloudformation delete-stack --stack-name logstashstack +echo "Waiting 60 minutes (press enter to continue)..." +read -t 3600 NullVariable + +echo "Deleting Opensearch stack..." +aws cloudformation delete-stack --stack-name opensearchteststack +echo "Waiting 60 minutes (press enter to continue)..." +read -t 3600 NullVariable + +# Create OpenSearch Cluster stack +echo "" +echo "Creating Opensearch stack..." +aws cloudformation create-stack --stack-name opensearchstack --template-body file://opensearch.yaml --parameters ParameterKey=SSHKey,ParameterValue=my-keypair --capabilities CAPABILITY_NAMED_IAM +echo "Waiting 60 minutes (press enter to continue)..." +read -t 3600 NullVariable + +# Create Logstash Cluster stack +aws cloudformation create-stack --stack-name logstashstack --template-body file://logstash_cluster.yaml --capabilities CAPABILITY_NAMED_IAM +``` + +Different changes to CloudFormatino templates have different "update behaviors". +For example, changing the EBS configuration of an OpenSearch cluster requires +"No interruption" ([source](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-opensearchservice-domain.html#cfn-opensearchservice-domain-ebsoptions)). +Others (like changing the name of a CloudFormation stack or a load balancer) require total replacement. +Those update behaviors are documented [here](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html). +If you're deploying a change that doesn't require resource replacement, you can run the "update-stack" commands: + +```shell +aws cloudformation update-stack --stack-name opensearchstack --template-body file://opensearch.yaml --parameters ParameterKey=SSHKey,ParameterValue=my-keypair --capabilities CAPABILITY_NAMED_IAM +aws cloudformation update-stack --stack-name logstashstack --template-body file://logstash_cluster.yaml --capabilities CAPABILITY_NAMED_IAM +``` + +#### 15.12.2021 + +What has been done: + +- creted cloudformation stack for Logstash, +- created Opensearch cluster, +- created readonly user and readonly role, +- deployed logscraper, loggearman-client and loggearman-worker services on logscraper01.openstack.org. + +Configuration has been describe `here `__. diff --git a/doc/source/ansible-role.rst b/doc/source/logscraper-role.rst similarity index 100% rename from doc/source/ansible-role.rst rename to doc/source/logscraper-role.rst diff --git a/doc/source/logsender-role.rst b/doc/source/logsender-role.rst new file mode 120000 index 0000000..90dd1fe --- /dev/null +++ b/doc/source/logsender-role.rst @@ -0,0 +1 @@ +../../ansible/roles/logsender/README.rst \ No newline at end of file diff --git a/doc/source/opensearch-configuration.md b/doc/source/opensearch-configuration.md new file mode 120000 index 0000000..65f6c62 --- /dev/null +++ b/doc/source/opensearch-configuration.md @@ -0,0 +1 @@ +../../opensearch-config/README.md \ No newline at end of file diff --git a/opensearch-config/README.md b/opensearch-config/README.md index 412b04d..f8fff46 100644 --- a/opensearch-config/README.md +++ b/opensearch-config/README.md @@ -1,44 +1,162 @@ -# About +# Opensearch configuration + +## About + This folder contains CloudFormation configurations for an AWS OpenSearch cluster and a set of Logstash servers behind a load balancer. -# Usage +## Usage + You'll need appropriate AWS permissions (to create and monitor resources). Put AWS credentials in `~/.aws/credentials` and run `deploy_opensearch.sh`. -# After Creation -OpenSearch users +## After Creation Opensearch -* Create a user with username 'logstash' and the entered password in OpenSearch, and assign it the "logstash" role. -* Create a user with username 'readonly' and password 'opensearch-readonly-PUBLIC-2021!' in OpenSearch, and grant it read-only privileges. Give it access to the Global tenant. +The Opensearch service requires additional configuration like creating readonly user, create logstash user etc. + +### Create user + +Users will be created in the Opensearch dashboards service. +We create only few internal users: + +* logstash - that will be used by logstash or logsender service +* readonly - readonly user that will be able to discover data, check visualization and dashboards +* openstack - readonly user with easy to remember password + +NOTE: +To skip `password_validation_regex` validation for user that should have easy to remember password, like `openstack` user, +it has been created via REST API. For example: + +```shell +bcrypt=$(htpasswd -bnBC 10 "" password | tr -d ':\n') +curl -X PUT "https:///_plugins/_security/api/internalusers/openstack" \ + -H 'Content-Type: application/json' \ + -d' { "hash" : "$2a$12$ABDOLV5fJDfXlkyNVAqD0O4AcUyvCV.Pq8jqLaPdHbsj0yRZYniNa" } ' \ + --user 'admin:myuserpassword' +``` + +### Creating roles + +Role will be added in the Opensearch dashboards service. +Created roles: + +* Readonly role is creaded base on the [inscruction](https://opensearch.org/docs/latest/security-plugin/access-control/users-roles/#set-up-a-read-only-user-in-opensearch-dashboards) +Details: + name: readonly + cluster permissions: cluster_composite_ops_ro + index permissions: + index: * + index permissions: read + tenant permissions: + tenant: global_tenant + +### Create role mapping + +After creating the role, inside the role you will be able to attach the user that should use it. + +## Create ILM - Index Lifecycle Management In the OpenSearch Dashboard select `Index Management`, `State management policies`, and then `Create Policy`. Make a policy with the following policy statement: -``` + +```json { - "policy_id": "DeleteAllDataAfter14Days", - "description": "Delete all data after 14 days", - "last_updated_time": 1639608774297, - "schema_version": 1, - "error_notification": null, - "default_state": "hot", - "states": [ - { - "name": "hot", - "actions": [], - "transitions": [ - { - "state_name": "delete", - "conditions": { - "min_index_age": "14d" + "policy": { + "description": "Delete all data after 14 days", + "default_state": "hot", + "states": [ + { + "name": "hot", + "actions": [], + "transitions": [ + { + "state_name": "delete", + "conditions": { + "min_index_age": "14d" + } } - } - ] - }, - { - "name": "delete", - "actions": [], - "transitions": [] - } - ], - "ism_template": null + ] + }, + { + "name": "delete", + "actions": [ + { + "delete": {} + } + ], + "transitions": [] + } + ] + } } ``` -This will delete all indices that are at least 7 days old (e.g. the `logstash-logs-2021.12.15` index will be deleted on 2021-12-22). + +This will delete all indices that are at least 14 days old (e.g. the `logstash-logs-2021.12.15` index will be deleted on 2021-12-22). + +## Advenced settings in Opensearch Dashboards + +There is only few changes applied comparing to default settings. +Differences in sections: + +* General + +> * Timezone for date formatting + +```shell +UTC +``` + +> * Default route: + +```shell +/app/discover?security_tenant=global +``` + +> * Time filter quick ranges: + +```json +[ + { + "from": "now/d", + "to": "now/d", + "display": "Today" + }, + { + "from": "now/w", + "to": "now/w", + "display": "This week" + }, + { + "from": "now-15m", + "to": "now", + "display": "Last 15 minutes" + }, + { + "from": "now-30m", + "to": "now", + "display": "Last 30 minutes" + }, + { + "from": "now-1h", + "to": "now", + "display": "Last 1 hour" + }, + { + "from": "now-6h", + "to": "now", + "display": "Last 6 hour" + }, + { + "from": "now-12h", + "to": "now", + "display": "Last 12 hour" + }, + { + "from": "now-24h", + "to": "now", + "display": "Last 24 hours" + }, + { + "from": "now-7d", + "to": "now", + "display": "Last 7 days" + } +] +``` diff --git a/setup.cfg b/setup.cfg index e69dc69..9e8f16b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,8 +1,8 @@ [metadata] name = logscraper summary = OpenStack CI Log Processing tool -description-file = - README.rst +description = "Get latest done logs from Zuul CI and push to Opensearch" +long_description = file: README.rst author = Openstack Contributors author-email = openstack-discuss@lists.openstack.org home-page = http://docs.openstack.org/infra/ci-log-processing