Add a seperate controller subnode to subdivide the memory load

The number of required plugins has increased to the extent that controller on Zuul FT infrastructure almost runs out of memory [1]. This potentially induces various problems such as FT failure noises or POST_FAILURE at the ansible task 'export-devstack-journal'. To request expanded images such as ubuntu-focal-expanded (16GB) or ubuntu-focal-32GB would be an option, but it turns out that would lead us to another problematic situation. [2] This patch, instead, addresses the issue by subdividing the memory load. As a first step, introduce a new subnode 'tacker-controller', on which both tacker-server and tacker-conductor are located. Note: * when we re-locate some other components to this new subnode, it might better to rename it. * `devstack_local_conf: {}` in .zuul.yaml is to cancel out the global job.vars devstack_local_conf.post-config.$NEUTRON_DHCP_CONF, which is not present on 'tacker-controller' in the first place. * TACKER_MODE is set to 'standalone'. 'all-in-one' supposes core services like nova, neutron, keystone, etc. api servers are located on the same host as tacker-server. * in devstack/lib/tacker:create_tacker_accounts, SERVICE_HOST should have been TACKER_HOST. this minor fix is included. * in roles/setup-default-vim/tasks/main.yaml, the same where conditions were scattered but all tasks in it just needed to run on 'controller' only. so let us wrap them all in a block. * renamed devstack/plugin.sh:tacker_register_default_vim for clarity. * policy file modification for Heat is now done by an ansible task. it frees us from the co-location requirements for Tacker and Heat. * drop devstack/lib/tacker:is_tacker_enabled as it's no longer needed. [1]: we investigated how severe the memory load on 'controller' was on Zuul FT infrastructure: * The highest memory-consuming processes in desc order: 808.70MB (9.87%) 828112 /usr/sbin/mysqld 179.81MB (2.19%) 184124 ... /usr/local/bin/tacker-server ... 152.57MB (1.86%) 156232 ... /usr/local/bin/tacker-conductor . 146.67MB (1.79%) 150188 ... /usr/local/bin/neutron-server ... 132.96MB (1.62%) 136148 ... /usr/local/bin/neutron-server ... 129.08MB (1.58%) 132180 ... /usr/local/bin/heat-engine ... 127.48MB (1.56%) 130544 ... /usr/local/bin/heat-engine ... 122.16MB (1.49%) 125092 nova-apiuWSGI worker 1 121.00MB (1.48%) 123900 neutron-openvswitch-agent ... 119.50MB (1.46%) 122368 cinder-apiuWSGI worker 1 ---(snip)--- * `free -m` output total used free shared buff/cache available Mem: 7955 7427 196 16 331 219 Swap: 1022 1019 3 [2]: http://eavesdrop.openstack.org/irclogs/%23openstack-infra/ %23openstack-infra.2020-11-25.log.html Change-Id: I030ffd5fd11b7ca9abca56e85e449ed4c4d709bd
2020-11-26 11:14:06 +09:00 · 2020-11-26 11:14:06 +09:00 · 578b12e989
commit 578b12e989
parent 6bbcad5af7
8 changed files with 164 additions and 121 deletions
--- a/.zuul.yaml
+++ b/.zuul.yaml
@ -1,8 +1,10 @@
 - nodeset:
-    name: openstack-3-nodes-focal
+    name: openstack-4-nodes-focal
    nodes:
      - name: controller
        label: ubuntu-focal
+      - name: controller-tacker
+        label: ubuntu-focal
      - name: compute1
        label: ubuntu-focal
      - name: compute2
@ -16,6 +18,7 @@
      # Nodes that are not the controller
      - name: subnode
        nodes:
+          - controller-tacker
          - compute1
          - compute2
      # Switch node for multinode networking setup
@ -25,6 +28,7 @@
      # Peer nodes for multinode networking setup
      - name: peers
        nodes:
+          - controller-tacker
          - compute1
          - compute2

@ -33,7 +37,7 @@
    parent: devstack
    description: |
      Base multinodes job for devstack-based functional tests
-    nodeset: openstack-3-nodes-focal
+    nodeset: openstack-4-nodes-focal
    pre-run: playbooks/devstack/pre.yaml
    run: playbooks/devstack/run.yaml
    post-run: playbooks/devstack/post.yaml
@ -70,8 +74,15 @@
    vars:
      devstack_localrc:
        CELLSV2_SETUP: singleconductor
+        PHYSICAL_NETWORK: mgmtphysnet0
+        OVS_BRIDGE_MAPPINGS: public:br-ex,mgmtphysnet0:br-infra
        Q_DVR_MODE: dvr
        DATABASE_TYPE: mysql
+        KEYSTONE_SERVICE_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}"
+        MYSQL_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}"
+        TACKER_HOST: "{{ hostvars['controller-tacker']['nodepool']['private_ipv4'] }}"
+        TACKER_MODE: standalone
+        IS_ZUUL_FT: True
        Q_SERVICE_PLUGIN_CLASSES: router,neutron.services.metering.metering_plugin.MeteringPlugin,networking_sfc.services.flowclassifier.plugin.FlowClassifierPlugin,neutron.services.qos.qos_plugin.QoSPlugin,qos
        Q_ML2_PLUGIN_EXT_DRIVERS: port_security,qos
        L2_AGENT_EXTENSIONS: qos
@ -88,13 +99,8 @@
          heat: https://opendev.org/openstack/heat
          networking-sfc: https://opendev.org/openstack/networking-sfc
          aodh: https://opendev.org/openstack/aodh
-          # Temporarily remove because of Ubuntu Focal Migration issue
-          #   https://storyboard.openstack.org/#!/story/2008121
-          #   https://review.opendev.org/#/c/752294/
-          # ceilometer: https://opendev.org/openstack/ceilometer
          barbican: https://opendev.org/openstack/barbican
          mistral: https://opendev.org/openstack/mistral
-          tacker: https://opendev.org/openstack/tacker
          blazar: https://opendev.org/openstack/blazar
          fenix: https://opendev.org/x/fenix
        devstack_services:
@ -131,9 +137,6 @@
          c-sch: true
          c-vol: true
          cinder: true
-          # Tacker services
-          tacker: true
-          tacker-conductor: true
          # Services we don't need.
          # This section is not really needed, it's for readability.
          horizon: false
@ -145,6 +148,19 @@
          s-proxy: false
          c-bak: false
        tox_install_siblings: false
+      controller-tacker:
+        devstack_local_conf: {}
+        devstack_services:
+          q-agt: true
+          # Tacker services
+          tacker: true
+          tacker-conductor: true
+        devstack_plugins:
+          # Temporarily remove because of Ubuntu Focal Migration issue
+          #   https://storyboard.openstack.org/#!/story/2008121
+          #   https://review.opendev.org/#/c/752294/
+          # ceilometer: https://opendev.org/openstack/ceilometer
+          tacker: https://opendev.org/openstack/tacker
        tox_envlist: dsvm-functional
    group-vars:
      subnode:
@ -152,7 +168,13 @@
          CELLSV2_SETUP: singleconductor
          PHYSICAL_NETWORK: mgmtphysnet0
          OVS_BRIDGE_MAPPINGS: public:br-ex,mgmtphysnet0:br-infra
+          Q_DVR_MODE: dvr
          DATABASE_TYPE: mysql
+          KEYSTONE_SERVICE_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}"
+          MYSQL_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}"
+          TACKER_HOST: "{{ hostvars['controller-tacker']['nodepool']['private_ipv4'] }}"
+          TACKER_MODE: standalone
+          IS_ZUUL_FT: True
        devstack_services:
          q-agt: true
          n-api: false
--- a/devstack/lib/tacker
+++ b/devstack/lib/tacker
@ -80,7 +80,6 @@ TACKER_NOVA_URL=${TACKER_NOVA_URL:-http://127.0.0.1:8774/v2}
 TACKER_NOVA_CA_CERTIFICATES_FILE=${TACKER_NOVA_CA_CERTIFICATES_FILE:-}
 TACKER_NOVA_API_INSECURE=${TACKER_NOVA_API_INSECURE:-False}

-HEAT_CONF_DIR=/etc/heat
 CEILOMETER_CONF_DIR=/etc/ceilometer

 source ${TACKER_DIR}/tacker/tests/contrib/post_test_hook_lib.sh
@ -91,13 +90,6 @@ GLANCE_DEFAULT_BACKEND=${GLANCE_DEFAULT_BACKEND:=file}

 # Functions
 # ---------
-# Test if any Tacker services are enabled
-# is_tacker_enabled
-function is_tacker_enabled {
-    [[ ,${ENABLED_SERVICES} =~ ,"tacker" ]] && return 0
-    return 1
-}
-
 # create_tacker_cache_dir() - Part of the _tacker_setup_keystone() process
 function create_tacker_cache_dir {
    # Create cache dir
@ -105,6 +97,28 @@ function create_tacker_cache_dir {
    rm -f $TACKER_AUTH_CACHE_DIR/*
 }

+# function install_db_client_mysql() - ensure mysql client is installed
+function install_db_client_mysql {
+    if is_oraclelinux; then
+        install_package mysql-community-client
+    elif is_fedora || is_suse; then
+        install_package mariadb-client
+    elif is_ubuntu; then
+        install_package mysql-client
+    else
+        exit_distro_not_supported "mysql client installation"
+    fi
+}
+
+# function install_db_client_postgresql() - ensure postgresql client is installed
+function install_db_client_postgresql {
+    if is_ubuntu || is_fedora || is_suse; then
+        install_package postgresql-client
+    else
+        exit_distro_not_supported "postgresql client installation"
+    fi
+}
+
 # create_tacker_accounts() - Set up common required tacker accounts

 # Tenant               User       Roles
@ -123,9 +137,9 @@ function create_tacker_accounts {
            "nfv-orchestration" "Tacker NFV Orchestration Service")
        get_or_create_endpoint $tacker_service \
            "$REGION_NAME" \
-            "$TACKER_PROTOCOL://$SERVICE_HOST:$TACKER_PORT/" \
-            "$TACKER_PROTOCOL://$SERVICE_HOST:$TACKER_PORT/" \
-            "$TACKER_PROTOCOL://$SERVICE_HOST:$TACKER_PORT/"
+            "$TACKER_PROTOCOL://$TACKER_HOST:$TACKER_PORT/" \
+            "$TACKER_PROTOCOL://$TACKER_HOST:$TACKER_PORT/" \
+            "$TACKER_PROTOCOL://$TACKER_HOST:$TACKER_PORT/"
    fi
 }

@ -134,6 +148,9 @@ function create_tacker_accounts {

 # init_tacker() - Initialize databases, etc.
 function init_tacker {
+    # In case db service is disabled, which would have installed cli commands.
+    install_db_client_$DATABASE_TYPE
+
    recreate_database $TACKER_DB_NAME

    # Run Tacker db migrations
@ -241,7 +258,7 @@ function configure_tacker {
    iniset $TACKER_CONF DEFAULT auth_strategy $TACKER_AUTH_STRATEGY
    _tacker_setup_keystone $TACKER_CONF keystone_authtoken

-    if [[ "${TACKER_MODE}" == "all" ]]; then
+    if [[ "${TACKER_MODE}" == "all" || -v IS_ZUUL_FT ]]; then
        iniset "/$Q_PLUGIN_CONF_FILE" ml2_type_flat flat_networks $PUBLIC_PHYSICAL_NETWORK,$MGMT_PHYS_NET
        iniset "/$Q_PLUGIN_CONF_FILE" ovs bridge_mappings $PUBLIC_PHYSICAL_NETWORK:$PUBLIC_BRIDGE,$MGMT_PHYS_NET:$BR_MGMT

@ -417,9 +434,9 @@ function tacker_create_initial_network {
    sudo ip address add ${NETWORK_GATEWAY_MGMT_IP} dev ${BR_MGMT}
 }

-function tacker_register_default_vim {
+function tacker_setup_default_vim_resources {
    # Note: These must be the same as in tacker/tests/etc/samples/local-vim.yaml
-    # and devstack/lib/tacker/vim_config.yaml
+    # and devstack/vim_config.yaml
    DEFAULT_VIM_PROJECT_NAME="nfv"
    DEFAULT_VIM_USER="nfv_user"
    DEFAULT_VIM_PASSWORD="devstack"
@ -475,13 +492,6 @@ function tacker_register_default_vim {

 }

-function modify_heat_flavor_policy_rule {
-    local policy_file=$HEAT_CONF_DIR/policy.yaml
-    touch $policy_file
-    # Allow non-admin projects with 'admin' roles to create flavors in Heat
-    echo '"resource_types:OS::Nova::Flavor": "role:admin"' >> $policy_file
-}
-
 function configure_maintenance_event_types {
    local event_definitions_file=$CEILOMETER_CONF_DIR/event_definitions.yaml
    local maintenance_events_file=$TACKER_DIR/etc/ceilometer/maintenance_event_types.yaml
--- a/devstack/plugin.sh
+++ b/devstack/plugin.sh
@ -32,15 +32,13 @@ if is_service_enabled tacker; then
            tacker_horizon_install
        fi

-        if [[ "${TACKER_MODE}" == "all" ]]; then
-            echo_summary "Modifying Heat policy.json file"
-            modify_heat_flavor_policy_rule
+        if [[ "${TACKER_MODE}" == "all" || -v IS_ZUUL_FT ]]; then
            echo_summary "Setup initial tacker network"
            tacker_create_initial_network
            echo_summary "Check and download images for tacker initial"
            tacker_check_and_download_images
-            echo_summary "Registering default VIM"
-            tacker_register_default_vim
+            echo_summary "Setup default VIM resources"
+            tacker_setup_default_vim_resources

            if is_service_enabled ceilometer; then
                echo_summary "Configure maintenance event types"
--- a/devstack/settings
+++ b/devstack/settings
@ -2,6 +2,28 @@ TACKER_MODE=${TACKER_MODE:-all}
 USE_BARBICAN=True
 KUBERNETES_VIM=${KUBERNETES_VIM:-False}

+if [ "${TACKER_MODE}" == "all" -o ${IS_ZUUL_FT+yes} ]; then
+    MGMT_PHYS_NET=${MGMT_PHYS_NET:-mgmtphysnet0}
+    # br-infra is created by devstack multi node job
+    # https://opendev.org/zuul/zuul-jobs/src/branch/master/roles/multi-node-bridge
+    BR_MGMT=${BR_MGMT:-br-infra}
+    NET_MGMT=${NET_MGMT:-net_mgmt}
+    SUBNET_MGMT=${SUBNET_MGMT:-subnet_mgmt}
+    FIXED_RANGE_MGMT=${FIXED_RANGE_MGMT:-192.168.120.0/24}
+    NETWORK_GATEWAY_MGMT=${NETWORK_GATEWAY_MGMT:-192.168.120.1}
+    NETWORK_GATEWAY_MGMT_IP=${NETWORK_GATEWAY_MGMT_IP:-192.168.120.1/24}
+
+    NET0=${NET0:-net0}
+    SUBNET0=${SUBNET0:-subnet0}
+    FIXED_RANGE0=${FIXED_RANGE0:-10.10.0.0/24}
+    NETWORK_GATEWAY0=${NETWORK_GATEWAY0:-10.10.0.1}
+
+    NET1=${NET1:-net1}
+    SUBNET1=${SUBNET1:-subnet1}
+    FIXED_RANGE1=${FIXED_RANGE1:-10.10.1.0/24}
+    NETWORK_GATEWAY1=${NETWORK_GATEWAY1:-10.10.1.1}
+fi
+
 if [ "${TACKER_MODE}" == "all" ]; then
    # Nova
    disable_service n-net
@ -21,25 +43,6 @@ if [ "${TACKER_MODE}" == "all" ]; then
    enable_service tacker
    # enable tacker-conductor will make systemctl enable conductor service
    enable_service tacker-conductor
-    MGMT_PHYS_NET=${MGMT_PHYS_NET:-mgmtphysnet0}
-    # br-infra is created by devstack multi node job
-    # https://opendev.org/zuul/zuul-jobs/src/branch/master/roles/multi-node-bridge
-    BR_MGMT=${BR_MGMT:-br-infra}
-    NET_MGMT=${NET_MGMT:-net_mgmt}
-    SUBNET_MGMT=${SUBNET_MGMT:-subnet_mgmt}
-    FIXED_RANGE_MGMT=${FIXED_RANGE_MGMT:-192.168.120.0/24}
-    NETWORK_GATEWAY_MGMT=${NETWORK_GATEWAY_MGMT:-192.168.120.1}
-    NETWORK_GATEWAY_MGMT_IP=${NETWORK_GATEWAY_MGMT_IP:-192.168.120.1/24}
-
-    NET0=${NET0:-net0}
-    SUBNET0=${SUBNET0:-subnet0}
-    FIXED_RANGE0=${FIXED_RANGE0:-10.10.0.0/24}
-    NETWORK_GATEWAY0=${NETWORK_GATEWAY0:-10.10.0.1}
-
-    NET1=${NET1:-net1}
-    SUBNET1=${SUBNET1:-subnet1}
-    FIXED_RANGE1=${FIXED_RANGE1:-10.10.1.0/24}
-    NETWORK_GATEWAY1=${NETWORK_GATEWAY1:-10.10.1.1}

    if [ "${KUBERNETES_VIM}" == "True" ]; then
        KURYR_NEUTRON_DEFAULT_PROJECT="default"
@ -59,8 +62,9 @@ if [ "${TACKER_MODE}" == "all" ]; then
        done
    fi
 elif [ "${TACKER_MODE}" == "standalone" ]; then
-    # set the enabled services here. This will need tacker devstack plugin put as the last one in local.conf
-    ENABLED_SERVICES=key,horizon,tacker,tacker-conductor,mysql,dstat,barbican,mistral,mistral-api,mistral-engine,mistral-executor,mistral-event-engine
+    enable_service dstat
+    enable_service tacker
+    enable_service tacker-conductor
 else
    die $LINENO "invalid value: $TACKER_MODE for TACKER_MODE"
 fi
--- a/playbooks/devstack/pre.yaml
+++ b/playbooks/devstack/pre.yaml
@ -1,6 +1,7 @@
 - hosts: all
  roles:
    - orchestrate-devstack
+    - modify-heat-policy
    - setup-default-vim
    - role: bindep
      bindep_profile: test
--- a/playbooks/devstack/run.yaml
+++ b/playbooks/devstack/run.yaml
@ -1,3 +1,3 @@
- hosts: controller
+- hosts: controller-tacker
  roles:
    - tox
--- a/roles/modify-heat-policy/tasks/main.yaml
+++ b/roles/modify-heat-policy/tasks/main.yaml
@ -0,0 +1,9 @@
+- name: Modify heat flavor policy rule
+  become: true
+  lineinfile:
+    path: /etc/heat/policy.yaml
+    regexp: '^"resource_types:OS::Nova::Flavor":'
+    line: '"resource_types:OS::Nova::Flavor": "role:admin"'
+    mode: 0644
+    create: true
+  when: devstack_plugins['heat']|default(false)
--- a/roles/setup-default-vim/tasks/main.yaml
+++ b/roles/setup-default-vim/tasks/main.yaml
@ -1,68 +1,67 @@
- name: Copy files test setup default vim sh
-  copy:
-     remote_src=True
-     src={{ devstack_base_dir }}/tacker/tools/test-setup-default-vim.sh
-     dest={{ zuul_work_dir }}/tools/test-setup-default-vim.sh
-     mode=0755
-  when: inventory_hostname == "controller"
+- block:
+  - name: Copy tools/test-setup-default-vim.sh
+    copy:
+       remote_src=True
+       src={{ devstack_base_dir }}/tacker/tools/test-setup-default-vim.sh
+       dest={{ zuul_work_dir }}/tools/test-setup-default-vim.sh
+       mode=0755

- name: Copy test vim file
-  copy:
-     remote_src=True
-     src={{ devstack_base_dir }}/tacker/tacker/tests/etc/samples/local-vim.yaml
-     dest={{ zuul_work_dir }}/tacker/tests/etc/samples/local-vim.yaml
-  when: inventory_hostname == "controller"
+  - name: Copy test vim file
+    copy:
+       remote_src=True
+       src={{ devstack_base_dir }}/tacker/tacker/tests/etc/samples/local-vim.yaml
+       dest={{ zuul_work_dir }}/tacker/tests/etc/samples/local-vim.yaml

- name: Check if project's tools/test-setup-default-vim.sh exists
-  stat:
-    path: "{{ zuul_work_dir }}/tools/test-setup-default-vim.sh"
-  register: p
- fail:
-    msg: >
-          {{ zuul_work_dir }}/tools/test-setup-default-vim.sh doesn't exists
-          or it doesn't have execute permission.
-  when:  p.stat.exists != True or p.stat.executable != True
+  - name: Check if project's tools/test-setup-default-vim.sh exists
+    stat:
+      path: "{{ zuul_work_dir }}/tools/test-setup-default-vim.sh"
+    register: p
+  - fail:
+      msg: >
+            {{ zuul_work_dir }}/tools/test-setup-default-vim.sh doesn't exists
+            or it doesn't have execute permission.
+    when:  p.stat.exists != True or p.stat.executable != True

- name: Get stackenv from devstack environment
-  slurp:
-    src: "{{ devstack_base_dir }}/devstack/.stackenv"
-  register: stackenv
+  - name: Get stackenv from devstack environment
+    slurp:
+      src: "{{ devstack_base_dir }}/devstack/.stackenv"
+    register: stackenv
+
+  - name: Set a keystone authentication uri
+    set_fact:
+      auth_uri: "{{
+                stackenv.content
+                | b64decode
+                | regex_replace('\n', ' ')
+                | regex_replace('^.*KEYSTONE_SERVICE_URI=([^ ]+).*$', '\\1')
+                }}"
+    when:
+      - p.stat.exists
+
+  - name: Replace auth uri in test-setup-default-vim.sh and local-vim.yaml
+    replace:
+       path: "{{ item }}"
+       regexp: "http://127.0.0.1/identity"
+       replace: "{{ auth_uri }}"
+    with_items:
+       - "{{ zuul_work_dir }}/tools/test-setup-default-vim.sh"
+       - "{{ zuul_work_dir }}/tacker/tests/etc/samples/local-vim.yaml"
+    when:
+      - p.stat.exists
+
+  - name: Replace the config file path in the test-setup-default-vim.sh
+    replace:
+       path: "{{ zuul_work_dir }}/tools/test-setup-default-vim.sh"
+       regexp: '(?<=config-file )([^ ]+)(?= )'
+       replace: "{{ ansible_env.HOME }}/{{ zuul_work_dir }}/tacker/tests/etc/samples/local-vim.yaml"
+
+  - name: Run tools/test-setup-default-vim.sh
+    command: tools/test-setup-default-vim.sh
+    args:
+      chdir: "{{ zuul_work_dir }}"
+    when:
+      - p.stat.exists
+      - p.stat.executable

- name: Set a keystone authentication uri
-  set_fact:
-    auth_uri: "{{
-              stackenv.content
-              | b64decode
-              | regex_replace('\n', ' ')
-              | regex_replace('^.*KEYSTONE_SERVICE_URI=([^ ]+).*$', '\\1')
-              }}"
  when:
-    - p.stat.exists
-
- name: Replace the keystone authentication uri in the test-setup-default-vim.sh and local-vim.yaml
-  replace:
-     path: "{{ item }}"
-     regexp: "http://127.0.0.1/identity"
-     replace: "{{ auth_uri }}"
-  with_items:
-     - "{{ zuul_work_dir }}/tools/test-setup-default-vim.sh"
-     - "{{ zuul_work_dir }}/tacker/tests/etc/samples/local-vim.yaml"
-  when:
-    - inventory_hostname == 'controller'
-    - p.stat.exists
-
- name: Replace the config file path in the test-setup-default-vim.sh
-  replace:
-     path: "{{ zuul_work_dir }}/tools/test-setup-default-vim.sh"
-     regexp: '(?<=config-file )([^ ]+)(?= )'
-     replace: "{{ ansible_env.HOME }}/{{ zuul_work_dir }}/tacker/tests/etc/samples/local-vim.yaml"
-  when: inventory_hostname == 'controller'
-
- name: Run tools/test-setup-default-vim.sh
-  command: tools/test-setup-default-vim.sh
-  args:
-    chdir: "{{ zuul_work_dir }}"
-  when:
-    - inventory_hostname == 'controller'
-    - p.stat.exists
-    - p.stat.executable
+    - inventory_hostname == 'controller-tacker'