From 21a8c7f905915d2c92a9a96a0f393f945af128bc Mon Sep 17 00:00:00 2001 From: James Slagle Date: Fri, 6 Apr 2018 17:24:08 -0400 Subject: [PATCH] Don't set deployments as hostvars This patch updates config-download to not create a vars file under group_vars for the server deployments. These variables end up as hostvars in Ansible, which has caused many escaping problems since hostvars are run through the templating engine. In combination with I06c9b3256567cc57d599bd4d9af617c718d2314f, these escaping problems should be eliminated. Additionally, by not using hostvars (set_fact), we can save memory since a hostvar is set on a host and persisted in memory. With many hosts and large deployments (such as all hiera data), this would have eventually been an issue. Instead, an Ansible lookup is used in deployments.yaml to read deployment data, which are now written to individual files by config-download. This also improves readability instead of having one big file of all deployment data. Change-Id: Ie09fc64cf85eb532f31684f1c9808b942672ff41 Closes-Bug: #1760990 Closes-Bug: #1763779 (cherry picked from commit 60f25d2bcd08d58b3bbbe63a1c9b9e156c5ab5cc) --- .../{group_var_server.j2 => deployment.j2} | 3 - tripleo_common/templates/deployments.yaml | 14 ++-- .../tests/utils/data/overcloud-controller-0 | 74 ------------------- .../ControllerHostEntryDeployment | 19 +++++ .../overcloud-controller-0/MyExtraConfigPost | 20 +++++ .../data/overcloud-controller-0/MyPostConfig | 19 +++++ .../overcloud-controller-0/NetworkDeployment | 19 +++++ .../tests/utils/data/overcloud-novacompute-0 | 54 -------------- .../ComputeHostEntryDeployment | 18 +++++ .../overcloud-novacompute-0/MyExtraConfigPost | 19 +++++ .../overcloud-novacompute-0/NetworkDeployment | 18 +++++ .../tests/utils/data/overcloud-novacompute-1 | 54 -------------- .../ComputeHostEntryDeployment | 18 +++++ .../overcloud-novacompute-1/MyExtraConfigPost | 20 +++++ .../overcloud-novacompute-1/NetworkDeployment | 18 +++++ .../tests/utils/data/overcloud-novacompute-2 | 74 ------------------- .../overcloud-novacompute-2/AnsibleDeployment | 22 ++++++ .../ComputeHostEntryDeployment | 18 +++++ .../overcloud-novacompute-2/MyExtraConfigPost | 19 +++++ .../overcloud-novacompute-2/NetworkDeployment | 18 +++++ tripleo_common/tests/utils/test_config.py | 56 ++++++++++++-- tripleo_common/utils/config.py | 27 +++++-- 22 files changed, 343 insertions(+), 278 deletions(-) rename tripleo_common/templates/{group_var_server.j2 => deployment.j2} (94%) delete mode 100644 tripleo_common/tests/utils/data/overcloud-controller-0 create mode 100644 tripleo_common/tests/utils/data/overcloud-controller-0/ControllerHostEntryDeployment create mode 100644 tripleo_common/tests/utils/data/overcloud-controller-0/MyExtraConfigPost create mode 100644 tripleo_common/tests/utils/data/overcloud-controller-0/MyPostConfig create mode 100644 tripleo_common/tests/utils/data/overcloud-controller-0/NetworkDeployment delete mode 100644 tripleo_common/tests/utils/data/overcloud-novacompute-0 create mode 100644 tripleo_common/tests/utils/data/overcloud-novacompute-0/ComputeHostEntryDeployment create mode 100644 tripleo_common/tests/utils/data/overcloud-novacompute-0/MyExtraConfigPost create mode 100644 tripleo_common/tests/utils/data/overcloud-novacompute-0/NetworkDeployment delete mode 100644 tripleo_common/tests/utils/data/overcloud-novacompute-1 create mode 100644 tripleo_common/tests/utils/data/overcloud-novacompute-1/ComputeHostEntryDeployment create mode 100644 tripleo_common/tests/utils/data/overcloud-novacompute-1/MyExtraConfigPost create mode 100644 tripleo_common/tests/utils/data/overcloud-novacompute-1/NetworkDeployment delete mode 100644 tripleo_common/tests/utils/data/overcloud-novacompute-2 create mode 100644 tripleo_common/tests/utils/data/overcloud-novacompute-2/AnsibleDeployment create mode 100644 tripleo_common/tests/utils/data/overcloud-novacompute-2/ComputeHostEntryDeployment create mode 100644 tripleo_common/tests/utils/data/overcloud-novacompute-2/MyExtraConfigPost create mode 100644 tripleo_common/tests/utils/data/overcloud-novacompute-2/NetworkDeployment diff --git a/tripleo_common/templates/group_var_server.j2 b/tripleo_common/templates/deployment.j2 similarity index 94% rename from tripleo_common/templates/group_var_server.j2 rename to tripleo_common/templates/deployment.j2 index a65a29ff9..1acc5a356 100644 --- a/tripleo_common/templates/group_var_server.j2 +++ b/tripleo_common/templates/deployment.j2 @@ -1,6 +1,5 @@ deploy_server_id: {{ server_id }} -{% for deployment in deployments %} {{ deployment.get('deployment_name') }}: {% if deployment.get('scalar') %} config: | @@ -28,5 +27,3 @@ deploy_server_id: {{ server_id }} description: {{ output.get('description') }} type: {{ output.get('type') }} {% endfor %} - -{% endfor %} diff --git a/tripleo_common/templates/deployments.yaml b/tripleo_common/templates/deployments.yaml index 019877d07..9cf8a06af 100644 --- a/tripleo_common/templates/deployments.yaml +++ b/tripleo_common/templates/deployments.yaml @@ -1,14 +1,16 @@ +- name: Lookup deployment UUID + set_fact: + deployment_uuid: "{{ lookup('file', role_name ~ '/' ~ ansible_hostname ~ '/' ~ item) | from_yaml | json_query(item ~ '.id')}}" + - name: "Render deployment file for {{ item }}" - template: - src: ../templates/heat-config.j2 + copy: + content: "[ {{ lookup('file', role_name ~ '/' ~ ansible_hostname ~ '/' ~ item) | from_yaml | json_query(item) }} ]" dest: "/var/lib/heat-config/tripleo-config-download/{{ item }}" become: true - vars: - deployment: "{{ vars[item] | to_json }}" - name: "Force remove deployed file for {{ item }}" file: - path: /var/lib/heat-config/deployed/{{ vars[item].id }}.json + path: /var/lib/heat-config/deployed/{{ deployment_uuid }}.json state: absent become: true when: force | bool @@ -16,7 +18,7 @@ - name: "Run deployment {{ item }}" shell: | /usr/libexec/os-refresh-config/configure.d/55-heat-config - exit $(jq .deploy_status_code /var/lib/heat-config/deployed/{{ vars[item].id }}.notify.json) + exit $(jq .deploy_status_code /var/lib/heat-config/deployed/{{ deployment_uuid }}.notify.json) become: true environment: HEAT_SHELL_CONFIG: /var/lib/heat-config/tripleo-config-download/{{ item }} diff --git a/tripleo_common/tests/utils/data/overcloud-controller-0 b/tripleo_common/tests/utils/data/overcloud-controller-0 deleted file mode 100644 index 427f99560..000000000 --- a/tripleo_common/tests/utils/data/overcloud-controller-0 +++ /dev/null @@ -1,74 +0,0 @@ -deploy_server_id: 00b3a5e1-5e8e-4b55-878b-2fa2271f15ad - -ControllerHostEntryDeployment: - config: | - #!/bin/bash - echo "A script" - creation_time: "None" - deployment_name: ControllerHostEntryDeployment - group: script - id: 50fcb6ee-4ff2-4318-b734-d3a7b45a8d6d - inputs: - - name: deploy_server_id - description: None - type: None - value: |- - 00b3a5e1-5e8e-4b55-878b-2fa2271f15ad - name: None - options: None - outputs: - -NetworkDeployment: - config: | - #!/bin/bash - echo "Another script" - creation_time: "None" - deployment_name: NetworkDeployment - group: script - id: 613b3009-7708-4b06-8c2a-56f8cd379aff - inputs: - - name: deploy_server_id - description: None - type: None - value: |- - 00b3a5e1-5e8e-4b55-878b-2fa2271f15ad - name: None - options: None - outputs: - -MyPostConfig: - config: | - #!/bin/bash - echo "A PostConfig script" - creation_time: "None" - deployment_name: MyPostConfig - group: script - id: ac63372f-bddd-45a0-aca9-6f0950d581e2 - inputs: - - name: deploy_server_id - description: None - type: None - value: |- - 00b3a5e1-5e8e-4b55-878b-2fa2271f15ad - name: None - options: None - outputs: - -MyExtraConfigPost: - config: | - #!/bin/bash - echo "An ExtraConfigPost script" - creation_time: "None" - deployment_name: MyExtraConfigPost - group: script - id: 9ccb2f66-7cd8-4e3e-a034-4c1cfafd037e - inputs: - - name: deploy_server_id - description: None - type: None - value: |- - 00b3a5e1-5e8e-4b55-878b-2fa2271f15ad - name: None - options: None - outputs: - diff --git a/tripleo_common/tests/utils/data/overcloud-controller-0/ControllerHostEntryDeployment b/tripleo_common/tests/utils/data/overcloud-controller-0/ControllerHostEntryDeployment new file mode 100644 index 000000000..ca17ebe0a --- /dev/null +++ b/tripleo_common/tests/utils/data/overcloud-controller-0/ControllerHostEntryDeployment @@ -0,0 +1,19 @@ +deploy_server_id: 00b3a5e1-5e8e-4b55-878b-2fa2271f15ad + +ControllerHostEntryDeployment: + config: | + #!/bin/bash + echo "A script" + creation_time: "None" + deployment_name: ControllerHostEntryDeployment + group: script + id: 50fcb6ee-4ff2-4318-b734-d3a7b45a8d6d + inputs: + - name: deploy_server_id + description: None + type: None + value: |- + 00b3a5e1-5e8e-4b55-878b-2fa2271f15ad + name: None + options: None + outputs: diff --git a/tripleo_common/tests/utils/data/overcloud-controller-0/MyExtraConfigPost b/tripleo_common/tests/utils/data/overcloud-controller-0/MyExtraConfigPost new file mode 100644 index 000000000..225aa31e4 --- /dev/null +++ b/tripleo_common/tests/utils/data/overcloud-controller-0/MyExtraConfigPost @@ -0,0 +1,20 @@ +deploy_server_id: 00b3a5e1-5e8e-4b55-878b-2fa2271f15ad + +MyExtraConfigPost: + config: | + #!/bin/bash + echo "An ExtraConfigPost script" + creation_time: "None" + deployment_name: MyExtraConfigPost + group: script + id: 9ccb2f66-7cd8-4e3e-a034-4c1cfafd037e + inputs: + - name: deploy_server_id + description: None + type: None + value: |- + 00b3a5e1-5e8e-4b55-878b-2fa2271f15ad + name: None + options: None + outputs: + diff --git a/tripleo_common/tests/utils/data/overcloud-controller-0/MyPostConfig b/tripleo_common/tests/utils/data/overcloud-controller-0/MyPostConfig new file mode 100644 index 000000000..191d42e90 --- /dev/null +++ b/tripleo_common/tests/utils/data/overcloud-controller-0/MyPostConfig @@ -0,0 +1,19 @@ +deploy_server_id: 00b3a5e1-5e8e-4b55-878b-2fa2271f15ad + +MyPostConfig: + config: | + #!/bin/bash + echo "A PostConfig script" + creation_time: "None" + deployment_name: MyPostConfig + group: script + id: ac63372f-bddd-45a0-aca9-6f0950d581e2 + inputs: + - name: deploy_server_id + description: None + type: None + value: |- + 00b3a5e1-5e8e-4b55-878b-2fa2271f15ad + name: None + options: None + outputs: diff --git a/tripleo_common/tests/utils/data/overcloud-controller-0/NetworkDeployment b/tripleo_common/tests/utils/data/overcloud-controller-0/NetworkDeployment new file mode 100644 index 000000000..e41f84aec --- /dev/null +++ b/tripleo_common/tests/utils/data/overcloud-controller-0/NetworkDeployment @@ -0,0 +1,19 @@ +deploy_server_id: 00b3a5e1-5e8e-4b55-878b-2fa2271f15ad + +NetworkDeployment: + config: | + #!/bin/bash + echo "Another script" + creation_time: "None" + deployment_name: NetworkDeployment + group: script + id: 613b3009-7708-4b06-8c2a-56f8cd379aff + inputs: + - name: deploy_server_id + description: None + type: None + value: |- + 00b3a5e1-5e8e-4b55-878b-2fa2271f15ad + name: None + options: None + outputs: diff --git a/tripleo_common/tests/utils/data/overcloud-novacompute-0 b/tripleo_common/tests/utils/data/overcloud-novacompute-0 deleted file mode 100644 index 0c2090ec6..000000000 --- a/tripleo_common/tests/utils/data/overcloud-novacompute-0 +++ /dev/null @@ -1,54 +0,0 @@ -deploy_server_id: a7db3010-a51f-4ae0-a791-2364d629d20d - -ComputeHostEntryDeployment: - config: - {"hiera_key1": "hiera_value1", "hiera_key2": "hiera_value2"} - creation_time: "None" - deployment_name: ComputeHostEntryDeployment - group: hiera - id: 26db1678-fe18-4acb-86db-19885fccae4c - inputs: - - name: deploy_server_id - description: None - type: None - value: |- - a7db3010-a51f-4ae0-a791-2364d629d20d - name: None - options: None - outputs: - -NetworkDeployment: - config: - {"hiera_key3": "hiera_value3", "hiera_key4": {"hiera_key4_subkey1": "hiera_key4_subvalue1", "hiera_key4_subkey0": "hiera_key4_subvalue0"}} - creation_time: "None" - deployment_name: NetworkDeployment - group: hiera - id: d21aefd1-9d94-44c0-b73a-16c8d4bc605c - inputs: - - name: deploy_server_id - description: None - type: None - value: |- - a7db3010-a51f-4ae0-a791-2364d629d20d - name: None - options: None - outputs: - -MyExtraConfigPost: - config: | - #!/bin/bash - echo "An ExtraConfigPost script" - creation_time: "None" - deployment_name: MyExtraConfigPost - group: script - id: 9ccb2f66-7cd8-4e3e-a034-4c1cfafd037e - inputs: - - name: deploy_server_id - description: None - type: None - value: |- - a7db3010-a51f-4ae0-a791-2364d629d20d - name: None - options: None - outputs: - diff --git a/tripleo_common/tests/utils/data/overcloud-novacompute-0/ComputeHostEntryDeployment b/tripleo_common/tests/utils/data/overcloud-novacompute-0/ComputeHostEntryDeployment new file mode 100644 index 000000000..04f2695df --- /dev/null +++ b/tripleo_common/tests/utils/data/overcloud-novacompute-0/ComputeHostEntryDeployment @@ -0,0 +1,18 @@ +deploy_server_id: a7db3010-a51f-4ae0-a791-2364d629d20d + +ComputeHostEntryDeployment: + config: + {"hiera_key1": "hiera_value1", "hiera_key2": "hiera_value2"} + creation_time: "None" + deployment_name: ComputeHostEntryDeployment + group: hiera + id: 26db1678-fe18-4acb-86db-19885fccae4c + inputs: + - name: deploy_server_id + description: None + type: None + value: |- + a7db3010-a51f-4ae0-a791-2364d629d20d + name: None + options: None + outputs: diff --git a/tripleo_common/tests/utils/data/overcloud-novacompute-0/MyExtraConfigPost b/tripleo_common/tests/utils/data/overcloud-novacompute-0/MyExtraConfigPost new file mode 100644 index 000000000..5762e826b --- /dev/null +++ b/tripleo_common/tests/utils/data/overcloud-novacompute-0/MyExtraConfigPost @@ -0,0 +1,19 @@ +deploy_server_id: a7db3010-a51f-4ae0-a791-2364d629d20d + +MyExtraConfigPost: + config: | + #!/bin/bash + echo "An ExtraConfigPost script" + creation_time: "None" + deployment_name: MyExtraConfigPost + group: script + id: 9ccb2f66-7cd8-4e3e-a034-4c1cfafd037e + inputs: + - name: deploy_server_id + description: None + type: None + value: |- + a7db3010-a51f-4ae0-a791-2364d629d20d + name: None + options: None + outputs: diff --git a/tripleo_common/tests/utils/data/overcloud-novacompute-0/NetworkDeployment b/tripleo_common/tests/utils/data/overcloud-novacompute-0/NetworkDeployment new file mode 100644 index 000000000..e66e3291e --- /dev/null +++ b/tripleo_common/tests/utils/data/overcloud-novacompute-0/NetworkDeployment @@ -0,0 +1,18 @@ +deploy_server_id: a7db3010-a51f-4ae0-a791-2364d629d20d + +NetworkDeployment: + config: + {"hiera_key3": "hiera_value3", "hiera_key4": {"hiera_key4_subkey1": "hiera_key4_subvalue1", "hiera_key4_subkey0": "hiera_key4_subvalue0"}} + creation_time: "None" + deployment_name: NetworkDeployment + group: hiera + id: d21aefd1-9d94-44c0-b73a-16c8d4bc605c + inputs: + - name: deploy_server_id + description: None + type: None + value: |- + a7db3010-a51f-4ae0-a791-2364d629d20d + name: None + options: None + outputs: diff --git a/tripleo_common/tests/utils/data/overcloud-novacompute-1 b/tripleo_common/tests/utils/data/overcloud-novacompute-1 deleted file mode 100644 index 90b759d8b..000000000 --- a/tripleo_common/tests/utils/data/overcloud-novacompute-1 +++ /dev/null @@ -1,54 +0,0 @@ -deploy_server_id: 8b07cd31-3083-4b88-a433-955f72039e2c - -ComputeHostEntryDeployment: - config: - {"hiera_key1": "hiera_value1", "hiera_key2": "hiera_value2"} - creation_time: "None" - deployment_name: ComputeHostEntryDeployment - group: hiera - id: 26db1678-fe18-4acb-86db-19885fccae4c - inputs: - - name: deploy_server_id - description: None - type: None - value: |- - 8b07cd31-3083-4b88-a433-955f72039e2c - name: None - options: None - outputs: - -NetworkDeployment: - config: - {"hiera_key3": "hiera_value3", "hiera_key4": {"hiera_key4_subkey1": "hiera_key4_subvalue1", "hiera_key4_subkey0": "hiera_key4_subvalue0"}} - creation_time: "None" - deployment_name: NetworkDeployment - group: hiera - id: d21aefd1-9d94-44c0-b73a-16c8d4bc605c - inputs: - - name: deploy_server_id - description: None - type: None - value: |- - 8b07cd31-3083-4b88-a433-955f72039e2c - name: None - options: None - outputs: - -MyExtraConfigPost: - config: | - #!/bin/bash - echo "An ExtraConfigPost script" - creation_time: "None" - deployment_name: MyExtraConfigPost - group: script - id: 9ccb2f66-7cd8-4e3e-a034-4c1cfafd037e - inputs: - - name: deploy_server_id - description: None - type: None - value: |- - 8b07cd31-3083-4b88-a433-955f72039e2c - name: None - options: None - outputs: - diff --git a/tripleo_common/tests/utils/data/overcloud-novacompute-1/ComputeHostEntryDeployment b/tripleo_common/tests/utils/data/overcloud-novacompute-1/ComputeHostEntryDeployment new file mode 100644 index 000000000..8bfa07176 --- /dev/null +++ b/tripleo_common/tests/utils/data/overcloud-novacompute-1/ComputeHostEntryDeployment @@ -0,0 +1,18 @@ +deploy_server_id: 8b07cd31-3083-4b88-a433-955f72039e2c + +ComputeHostEntryDeployment: + config: + {"hiera_key1": "hiera_value1", "hiera_key2": "hiera_value2"} + creation_time: "None" + deployment_name: ComputeHostEntryDeployment + group: hiera + id: 26db1678-fe18-4acb-86db-19885fccae4c + inputs: + - name: deploy_server_id + description: None + type: None + value: |- + 8b07cd31-3083-4b88-a433-955f72039e2c + name: None + options: None + outputs: diff --git a/tripleo_common/tests/utils/data/overcloud-novacompute-1/MyExtraConfigPost b/tripleo_common/tests/utils/data/overcloud-novacompute-1/MyExtraConfigPost new file mode 100644 index 000000000..e130e5700 --- /dev/null +++ b/tripleo_common/tests/utils/data/overcloud-novacompute-1/MyExtraConfigPost @@ -0,0 +1,20 @@ +deploy_server_id: 8b07cd31-3083-4b88-a433-955f72039e2c + +MyExtraConfigPost: + config: | + #!/bin/bash + echo "An ExtraConfigPost script" + creation_time: "None" + deployment_name: MyExtraConfigPost + group: script + id: 9ccb2f66-7cd8-4e3e-a034-4c1cfafd037e + inputs: + - name: deploy_server_id + description: None + type: None + value: |- + 8b07cd31-3083-4b88-a433-955f72039e2c + name: None + options: None + outputs: + diff --git a/tripleo_common/tests/utils/data/overcloud-novacompute-1/NetworkDeployment b/tripleo_common/tests/utils/data/overcloud-novacompute-1/NetworkDeployment new file mode 100644 index 000000000..879186143 --- /dev/null +++ b/tripleo_common/tests/utils/data/overcloud-novacompute-1/NetworkDeployment @@ -0,0 +1,18 @@ +deploy_server_id: 8b07cd31-3083-4b88-a433-955f72039e2c + +NetworkDeployment: + config: + {"hiera_key3": "hiera_value3", "hiera_key4": {"hiera_key4_subkey1": "hiera_key4_subvalue1", "hiera_key4_subkey0": "hiera_key4_subvalue0"}} + creation_time: "None" + deployment_name: NetworkDeployment + group: hiera + id: d21aefd1-9d94-44c0-b73a-16c8d4bc605c + inputs: + - name: deploy_server_id + description: None + type: None + value: |- + 8b07cd31-3083-4b88-a433-955f72039e2c + name: None + options: None + outputs: diff --git a/tripleo_common/tests/utils/data/overcloud-novacompute-2 b/tripleo_common/tests/utils/data/overcloud-novacompute-2 deleted file mode 100644 index 791cacf32..000000000 --- a/tripleo_common/tests/utils/data/overcloud-novacompute-2 +++ /dev/null @@ -1,74 +0,0 @@ -deploy_server_id: 169b46f8-1965-4d90-a7de-f36fb4a830fe - -ComputeHostEntryDeployment: - config: - {"hiera_key1": "hiera_value1", "hiera_key2": "hiera_value2"} - creation_time: "None" - deployment_name: ComputeHostEntryDeployment - group: hiera - id: 26db1678-fe18-4acb-86db-19885fccae4c - inputs: - - name: deploy_server_id - description: None - type: None - value: |- - 169b46f8-1965-4d90-a7de-f36fb4a830fe - name: None - options: None - outputs: - -NetworkDeployment: - config: - {"hiera_key3": "hiera_value3", "hiera_key4": {"hiera_key4_subkey1": "hiera_key4_subvalue1", "hiera_key4_subkey0": "hiera_key4_subvalue0"}} - creation_time: "None" - deployment_name: NetworkDeployment - group: hiera - id: d21aefd1-9d94-44c0-b73a-16c8d4bc605c - inputs: - - name: deploy_server_id - description: None - type: None - value: |- - 169b46f8-1965-4d90-a7de-f36fb4a830fe - name: None - options: None - outputs: - -MyExtraConfigPost: - config: | - #!/bin/bash - echo "An ExtraConfigPost script" - creation_time: "None" - deployment_name: MyExtraConfigPost - group: script - id: 9ccb2f66-7cd8-4e3e-a034-4c1cfafd037e - inputs: - - name: deploy_server_id - description: None - type: None - value: |- - 169b46f8-1965-4d90-a7de-f36fb4a830fe - name: None - options: None - outputs: - -AnsibleDeployment: - id: 391263ba-3d06-43b6-a47e-481808aaff20 - creation_time: "None" - deployment_name: AnsibleDeployment - name: None - options: None - group: ansible - inputs: - - name: deploy_server_id - description: None - type: None - value: |- - 169b46f8-1965-4d90-a7de-f36fb4a830fe - outputs: - config: | - tasks: - - name: An Ansible task - copy: - content: "{{ some_hostvar | to_json }}" - dest: /some/path diff --git a/tripleo_common/tests/utils/data/overcloud-novacompute-2/AnsibleDeployment b/tripleo_common/tests/utils/data/overcloud-novacompute-2/AnsibleDeployment new file mode 100644 index 000000000..db1ab9ccd --- /dev/null +++ b/tripleo_common/tests/utils/data/overcloud-novacompute-2/AnsibleDeployment @@ -0,0 +1,22 @@ +deploy_server_id: 169b46f8-1965-4d90-a7de-f36fb4a830fe + +AnsibleDeployment: + id: 391263ba-3d06-43b6-a47e-481808aaff20 + creation_time: "None" + deployment_name: AnsibleDeployment + name: None + options: None + group: ansible + inputs: + - name: deploy_server_id + description: None + type: None + value: |- + 169b46f8-1965-4d90-a7de-f36fb4a830fe + outputs: + config: | + tasks: + - name: An Ansible task + copy: + content: "{{ some_hostvar | to_json }}" + dest: /some/path diff --git a/tripleo_common/tests/utils/data/overcloud-novacompute-2/ComputeHostEntryDeployment b/tripleo_common/tests/utils/data/overcloud-novacompute-2/ComputeHostEntryDeployment new file mode 100644 index 000000000..bdaeb2623 --- /dev/null +++ b/tripleo_common/tests/utils/data/overcloud-novacompute-2/ComputeHostEntryDeployment @@ -0,0 +1,18 @@ +deploy_server_id: 169b46f8-1965-4d90-a7de-f36fb4a830fe + +ComputeHostEntryDeployment: + config: + {"hiera_key1": "hiera_value1", "hiera_key2": "hiera_value2"} + creation_time: "None" + deployment_name: ComputeHostEntryDeployment + group: hiera + id: 26db1678-fe18-4acb-86db-19885fccae4c + inputs: + - name: deploy_server_id + description: None + type: None + value: |- + 169b46f8-1965-4d90-a7de-f36fb4a830fe + name: None + options: None + outputs: diff --git a/tripleo_common/tests/utils/data/overcloud-novacompute-2/MyExtraConfigPost b/tripleo_common/tests/utils/data/overcloud-novacompute-2/MyExtraConfigPost new file mode 100644 index 000000000..026cc492b --- /dev/null +++ b/tripleo_common/tests/utils/data/overcloud-novacompute-2/MyExtraConfigPost @@ -0,0 +1,19 @@ +deploy_server_id: 169b46f8-1965-4d90-a7de-f36fb4a830fe + +MyExtraConfigPost: + config: | + #!/bin/bash + echo "An ExtraConfigPost script" + creation_time: "None" + deployment_name: MyExtraConfigPost + group: script + id: 9ccb2f66-7cd8-4e3e-a034-4c1cfafd037e + inputs: + - name: deploy_server_id + description: None + type: None + value: |- + 169b46f8-1965-4d90-a7de-f36fb4a830fe + name: None + options: None + outputs: diff --git a/tripleo_common/tests/utils/data/overcloud-novacompute-2/NetworkDeployment b/tripleo_common/tests/utils/data/overcloud-novacompute-2/NetworkDeployment new file mode 100644 index 000000000..8efe61915 --- /dev/null +++ b/tripleo_common/tests/utils/data/overcloud-novacompute-2/NetworkDeployment @@ -0,0 +1,18 @@ +deploy_server_id: 169b46f8-1965-4d90-a7de-f36fb4a830fe + +NetworkDeployment: + config: + {"hiera_key3": "hiera_value3", "hiera_key4": {"hiera_key4_subkey1": "hiera_key4_subvalue1", "hiera_key4_subkey0": "hiera_key4_subvalue0"}} + creation_time: "None" + deployment_name: NetworkDeployment + group: hiera + id: d21aefd1-9d94-44c0-b73a-16c8d4bc605c + inputs: + - name: deploy_server_id + description: None + type: None + value: |- + 169b46f8-1965-4d90-a7de-f36fb4a830fe + name: None + options: None + outputs: diff --git a/tripleo_common/tests/utils/test_config.py b/tripleo_common/tests/utils/test_config.py index b71c948d1..4f0fd2e7a 100644 --- a/tripleo_common/tests/utils/test_config.py +++ b/tripleo_common/tests/utils/test_config.py @@ -277,11 +277,7 @@ class TestConfig(base.TestCase): self.tmp_dir = self.useFixture(fixtures.TempDir()).path tmp_path = self.config.download_config(stack, self.tmp_dir) - for f in ['overcloud-controller-0', - 'overcloud-novacompute-0', - 'overcloud-novacompute-1', - 'overcloud-novacompute-2', - 'Controller', + for f in ['Controller', 'Compute', ]: self.assertEqual( @@ -289,6 +285,56 @@ class TestConfig(base.TestCase): open(os.path.join(tmp_path, 'group_vars', f)).read()), self._get_yaml_file(f)) + for d in ['ControllerHostEntryDeployment', + 'NetworkDeployment', + 'MyExtraConfigPost', + 'MyPostConfig']: + self.assertEqual( + yaml.safe_load( + open(os.path.join(tmp_path, 'Controller', + 'overcloud-controller-0', + d)).read()), + self._get_yaml_file(os.path.join( + 'overcloud-controller-0', + d))) + + for d in ['ComputeHostEntryDeployment', + 'NetworkDeployment', + 'MyExtraConfigPost']: + self.assertEqual( + yaml.safe_load( + open(os.path.join(tmp_path, 'Compute', + 'overcloud-novacompute-0', + d)).read()), + self._get_yaml_file(os.path.join( + 'overcloud-novacompute-0', + d))) + + for d in ['ComputeHostEntryDeployment', + 'NetworkDeployment', + 'MyExtraConfigPost']: + self.assertEqual( + yaml.safe_load( + open(os.path.join(tmp_path, 'Compute', + 'overcloud-novacompute-1', + d)).read()), + self._get_yaml_file(os.path.join( + 'overcloud-novacompute-1', + d))) + + for d in ['ComputeHostEntryDeployment', + 'NetworkDeployment', + 'MyExtraConfigPost', + 'AnsibleDeployment']: + self.assertEqual( + yaml.safe_load( + open(os.path.join(tmp_path, 'Compute', + 'overcloud-novacompute-2', + d)).read()), + self._get_yaml_file(os.path.join( + 'overcloud-novacompute-2', + d))) + @patch('tripleo_common.utils.config.Config.get_config_dict') @patch('tripleo_common.utils.config.Config.get_deployment_data') def test_config_download_os_apply_config( diff --git a/tripleo_common/utils/config.py b/tripleo_common/utils/config.py index c77968803..64866c366 100644 --- a/tripleo_common/utils/config.py +++ b/tripleo_common/utils/config.py @@ -137,7 +137,7 @@ class Config(object): def _mkdir(self, dirname): if not os.path.exists(dirname): try: - os.mkdir(dirname, 0o700) + os.makedirs(dirname, 0o700) except OSError as e: message = 'Failed to create: %s, error: %s' % (dirname, str(e)) @@ -211,6 +211,9 @@ class Config(object): # that role. The deployment names are futher separated in their own # dict with keys of pre_deployment/post_deployment. role_deployment_names = {} + # server_roles is a dict of server name to server role for easier + # lookup + server_roles = {} for deployment in deployments_data: server_id = deployment.attributes['value']['server'] @@ -241,6 +244,8 @@ class Config(object): role_post_deployments = role_deployments.setdefault( 'post_deployments', []) + server_roles[server_names[server_id]] = role + # special handling of deployments that are run post the deploy # steps. We have to look these up based on the # physical_resource_id, but these names should be consistent since @@ -264,11 +269,16 @@ class Config(object): self._mkdir(group_vars_dir) for server, deployments in server_deployments.items(): - group_var_server_path = os.path.join(group_vars_dir, server) - group_var_server_template = env.get_template('group_var_server.j2') + deployment_template = env.get_template('deployment.j2') for d in deployments: + server_deployment_dir = os.path.join( + tmp_path, server_roles[server], server) + self._mkdir(server_deployment_dir) + deployment_path = os.path.join( + server_deployment_dir, d['deployment_name']) + # See if the config can be loaded as a JSON data structure # In some cases, it may already be JSON (hiera), or it may just # be a string (script). In those cases, just use the value @@ -280,7 +290,8 @@ class Config(object): # If the value is not a string already, pretty print it as a # string so it's rendered in a readable format. - if not isinstance(data, six.text_type): + if not (isinstance(data, six.text_type) or + isinstance(data, six.string_types)): data = json.dumps(data, indent=2) d['config'] = data @@ -298,10 +309,10 @@ class Config(object): "config-download." % d['deployment_name']) warnings.warn(message, DeprecationWarning) - with open(group_var_server_path, 'w') as f: - f.write(group_var_server_template.render( - deployments=deployments, - server_id=server_ids[server])) + with open(deployment_path, 'w') as f: + f.write(deployment_template.render( + deployment=d, + server_id=server_ids[server])) for role, deployments in role_deployment_names.items(): group_var_role_path = os.path.join(group_vars_dir, role)