Browse Source

(fix) Use endpoint for MAAS URL

- Instead of forcing a user to provide the full URL for the MAAS API,
  instead use the endpoints pattern and render the URI via HTK templates.
- Add secret name to chart to support HTK ingress
- Install libyaml to take advantage of faster parsing by pyyaml
- Add exception logging when node compiling fails.
- Add caching of parsed design to gain efficiency
- Add TLS certificate secret for use by the ingress document

Change-Id: I5a2dbc415483c336d38d67edcebdfc5812f7bb0c
Scott Hussey 5 months ago
parent
commit
a2418241ce

+ 5
- 0
charts/drydock/templates/configmap-etc.yaml View File

@@ -22,6 +22,11 @@
22 22
 {{- tuple "postgresql" "internal" "user" "postgresql" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | set .Values.conf.drydock.database "database_connect_string" | quote | trunc 0 -}}
23 23
 {{- end -}}
24 24
 
25
+# Render MAAS connection string if it is not explicitly configured
26
+{{- if empty .Values.conf.drydock.maasdriver.maas_api_url -}}
27
+{{- tuple "maas_region" "internal" "region_api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.drydock.maasdriver "maas_api_url" | quote | trunc 0 -}}
28
+{{- end -}}
29
+
25 30
 {{- if empty .Values.conf.drydock.keystone_authtoken.auth_uri -}}
26 31
 {{- tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.drydock.keystone_authtoken "auth_uri" | quote | trunc 0 -}}
27 32
 {{- end -}}

+ 18
- 0
charts/drydock/templates/secret-tls.yaml View File

@@ -0,0 +1,18 @@
1
+{{/*
2
+# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
3
+#
4
+# Licensed under the Apache License, Version 2.0 (the "License");
5
+# you may not use this file except in compliance with the License.
6
+# You may obtain a copy of the License at
7
+#
8
+#     http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+# Unless required by applicable law or agreed to in writing, software
11
+# distributed under the License is distributed on an "AS IS" BASIS,
12
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+# See the License for the specific language governing permissions and
14
+# limitations under the License.
15
+*/}}
16
+{{- if .Values.manifests.secret_tls }}
17
+{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "physicalprovisioner" ) -}}
18
+{{- end }}

+ 35
- 3
charts/drydock/values.yaml View File

@@ -131,6 +131,7 @@ manifests:
131 131
   secret_keystone: true
132 132
   secret_database: true
133 133
   secret_ssh_key: true
134
+  secret_tls: true
134 135
   configmap_etc: true
135 136
   configmap_bin: true
136 137
   service_drydock: true
@@ -234,7 +235,12 @@ endpoints:
234 235
       default: http
235 236
     host_fqdn_override:
236 237
       default: null
237
-      public: drydock.test.local
238
+      public:
239
+        host: drydock.test.local
240
+#        tls:
241
+#          crt: replace
242
+#          ca: replace
243
+#          key: replace
238 244
   postgresql:
239 245
     name: postgresql
240 246
     auth:
@@ -254,7 +260,29 @@ endpoints:
254 260
         default: 5432
255 261
     host_fqdn_override:
256 262
       default: null
257
-
263
+  maas_region:
264
+    name: maas-region
265
+    auth:
266
+      admin:
267
+        username: admin
268
+        password: admin
269
+        email: none@none
270
+    hosts:
271
+      default: maas-region
272
+      public: maas
273
+    path:
274
+      default: /MAAS
275
+    scheme:
276
+      default: http
277
+    port:
278
+      region_api:
279
+        default: 80
280
+        public: 80
281
+        nodeport: 31900
282
+      region_proxy:
283
+        default: 8000
284
+    host_fqdn_override:
285
+      default: null
258 286
 secrets:
259 287
   identity:
260 288
     admin: drydock-keystone-admin
@@ -263,6 +291,10 @@ secrets:
263 291
     admin: drydock-postgresql-admin
264 292
     user: drydock-postgresql-user
265 293
   ssh_key: ssh-private-key
294
+  tls:
295
+    physicalprovisioner:
296
+      api:
297
+        public: drydock-tls-public
266 298
 
267 299
 # Settings for drydock.conf
268 300
 conf:
@@ -288,7 +320,7 @@ conf:
288 320
       control_logger_name: '${global_logger_name}.control'
289 321
     maasdriver:
290 322
       maas_api_key: 'override_this'
291
-      maas_api_url: 'override_this'
323
+      maas_api_url: null
292 324
     plugins:
293 325
       ingester:
294 326
         - 'drydock_provisioner.ingester.plugins.yaml.YamlIngester'

+ 2
- 1
python/drydock_provisioner/drivers/node/maasdriver/api_client.py View File

@@ -54,7 +54,8 @@ class MaasOauth(req_auth.AuthBase):
54 54
 
55 55
 class MaasRequestFactory(object):
56 56
     def __init__(self, base_url, apikey):
57
-        self.base_url = base_url
57
+        # The URL in the config should end in /MAAS/, but the api is behind /MAAS/api/2.0/
58
+        self.base_url = base_url + "/api/2.0/"
58 59
         self.apikey = apikey
59 60
 
60 61
         self.signer = MaasOauth(apikey)

+ 23
- 1
python/drydock_provisioner/ingester/plugins/deckhand.py View File

@@ -19,13 +19,23 @@ import jsonschema
19 19
 import os
20 20
 import pkg_resources
21 21
 import copy
22
+import hashlib
22 23
 
23 24
 import drydock_provisioner.objects.fields as hd_fields
24 25
 
26
+from beaker.cache import CacheManager
27
+from beaker.util import parse_cache_config_options
28
+
25 29
 from drydock_provisioner import error as errors
26 30
 from drydock_provisioner import objects
27 31
 from drydock_provisioner.ingester.plugins import IngesterPlugin
28 32
 
33
+cache_opts = {
34
+    'cache.type': 'memory',
35
+    'expire': 1800,
36
+}
37
+
38
+cache = CacheManager(**parse_cache_config_options(cache_opts))
29 39
 
30 40
 class DeckhandIngester(IngesterPlugin):
31 41
     def __init__(self):
@@ -44,8 +54,20 @@ class DeckhandIngester(IngesterPlugin):
44 54
 
45 55
         :returns: a tuple of a status response and a list of parsed objects from drydock_provisioner.objects
46 56
         """
57
+        def local_parse():
58
+            return self.parse_docs(kwargs.get('content'))
59
+
47 60
         if 'content' in kwargs:
48
-            parse_status, models = self.parse_docs(kwargs.get('content'))
61
+            try:
62
+                # Hash the input to use as the cache key. This is not a security
63
+                # related hash, so use cheap and fast MD5
64
+                hv = hashlib.md5(kwargs.get('content', b'')).hexdigest()
65
+                local_cache = cache.get_cache('parsed_docs')
66
+                results = local_cache.get(key=hv, createfunc=local_parse)
67
+                parse_status, models = results
68
+            except Exception as ex:
69
+                self.logger.debug("Error parsing design - hash %s", hv, exc_info=ex)
70
+                raise ex
49 71
         else:
50 72
             raise ValueError('Missing parameter "content"')
51 73
 

+ 3
- 3
python/drydock_provisioner/orchestrator/orchestrator.py View File

@@ -269,10 +269,10 @@ class Orchestrator(object):
269 269
                         site_design,
270 270
                         state_manager=self.state_manager,
271 271
                         resolve_aliases=resolve_aliases)
272
-                except Exception:
272
+                except Exception as ex:
273 273
                     node_failed.append(n)
274
-                    self.logger.error(
275
-                        "Failed to build applied model for node %s." % n.name)
274
+                    self.logger.debug(
275
+                        "Failed to build applied model for node %s.", n.name, exc_info=ex)
276 276
             if node_failed:
277 277
                 raise errors.DesignError(
278 278
                     "Failed to build applied model for %s" % ",".join(

+ 1
- 1
python/drydock_provisioner/statemgmt/design/resolver.py View File

@@ -28,7 +28,7 @@ from drydock_provisioner.config import config_mgr
28 28
 
29 29
 cache_opts = {
30 30
     'cache.type': 'memory',
31
-    'expire': 300,
31
+    'expire': 180,
32 32
 }
33 33
 
34 34
 cache = CacheManager(**parse_cache_config_options(cache_opts))

+ 12
- 18
python/tests/unit/test_api_nodes_unit.py View File

@@ -13,7 +13,6 @@
13 13
 # limitations under the License.
14 14
 """Test Nodes API"""
15 15
 from falcon import testing
16
-from unittest.mock import Mock
17 16
 
18 17
 import pytest
19 18
 import json
@@ -85,20 +84,15 @@ class TestNodesApiUnit(object):
85 84
 
86 85
 
87 86
 @pytest.fixture()
88
-def mock_process_node_filter(deckhand_orchestrator):
89
-    def side_effect(**kwargs):
90
-        n1 = objects.BaremetalNode()
91
-        n1.name = 'n1'
92
-        n1.site = 'test1'
93
-        n2 = objects.BaremetalNode()
94
-        n2.name = 'n2'
95
-        n2.site = 'test2'
96
-        return [n1, n2]
97
-
98
-    deckhand_orchestrator.real_process_node_filter = deckhand_orchestrator.process_node_filter
99
-    deckhand_orchestrator.process_node_filter = Mock(side_effect=side_effect)
100
-
101
-    yield
102
-    deckhand_orchestrator.process_node_filter = Mock(
103
-        wraps=None, side_effect=None)
104
-    deckhand_orchestrator.process_node_filter = deckhand_orchestrator.real_process_node_filter
87
+def mock_process_node_filter(mocker, deckhand_orchestrator):
88
+    n1 = objects.BaremetalNode()
89
+    n1.name = 'n1'
90
+    n1.site = 'test1'
91
+    n2 = objects.BaremetalNode()
92
+    n2.name = 'n2'
93
+    n2.site = 'test2'
94
+    mock_results = [n1, n2]
95
+
96
+    with mocker.patch('drydock_provisioner.orchestrator.orchestrator.Orchestrator.process_node_filter',
97
+                      mocker.MagicMock(return_value=mock_results)):
98
+        yield

+ 4
- 4
python/tests/unit/test_api_validation.py View File

@@ -27,7 +27,7 @@ LOG = logging.getLogger(__name__)
27 27
 
28 28
 
29 29
 class TestValidationApi(object):
30
-    def test_post_validation_resp(self, input_files, falcontest, drydock_state,
30
+    def test_post_validation_resp(self, setup_logging, input_files, falcontest, drydock_state,
31 31
                                   mock_get_build_data):
32 32
 
33 33
         input_file = input_files.join("deckhand_fullsite.yaml")
@@ -52,7 +52,7 @@ class TestValidationApi(object):
52 52
         LOG.debug(result.text)
53 53
         assert result.status == falcon.HTTP_200
54 54
 
55
-    def test_href_error(self, input_files, falcontest):
55
+    def test_href_error(self, setup_logging, input_files, falcontest):
56 56
         url = '/api/v1.0/validatedesign'
57 57
         hdr = {
58 58
             'Content-Type': 'application/json',
@@ -72,7 +72,7 @@ class TestValidationApi(object):
72 72
         LOG.debug(result.text)
73 73
         assert result.status == falcon.HTTP_400
74 74
 
75
-    def test_json_data_error(self, input_files, falcontest):
75
+    def test_json_data_error(self, setup_logging, input_files, falcontest):
76 76
         url = '/api/v1.0/validatedesign'
77 77
         hdr = {
78 78
             'Content-Type': 'application/json',
@@ -88,7 +88,7 @@ class TestValidationApi(object):
88 88
         LOG.debug(result.text)
89 89
         assert result.status == falcon.HTTP_400
90 90
 
91
-    def test_invalid_post_resp(self, input_files, falcontest, drydock_state,
91
+    def test_invalid_post_resp(self, setup_logging, input_files, falcontest, drydock_state,
92 92
                                mock_get_build_data):
93 93
         input_file = input_files.join("invalid_validation.yaml")
94 94
         design_ref = "file://%s" % str(input_file)

+ 1
- 0
requirements-host.txt View File

@@ -1,6 +1,7 @@
1 1
 # These are host packages needed for Drydock
2 2
 # that don't come on a minimal Ubuntu install
3 3
 libvirt-dev
4
+libyaml-dev
4 5
 pkg-config
5 6
 python3-dev
6 7
 gcc

+ 1
- 1
tools/helm_tk.sh View File

@@ -18,7 +18,7 @@
18 18
 HELM=$1
19 19
 HTK_REPO=${HTK_REPO:-"https://github.com/openstack/openstack-helm-infra"}
20 20
 HTK_PATH=${HTK_PATH:-""}
21
-HTK_STABLE_COMMIT=${HTK_COMMIT:-"2fce7e821201a5f578331370703e164d5a932fcc"}
21
+HTK_STABLE_COMMIT=${HTK_COMMIT:-"42249d4243d892b7d04ce6aed9b0c8d7edcbfc7a"}
22 22
 DEP_UP_LIST=${DEP_UP_LIST:-"drydock"}
23 23
 BUILD_DIR=${BUILD_DIR:-$(mktemp -d)}
24 24
 

Loading…
Cancel
Save