Merge "(fix) Use endpoint for MAAS URL"

charts/drydock/templates/configmap-etc.yaml

@@ -22,6 +22,11 @@
 {{- tuple "postgresql" "internal" "user" "postgresql" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | set .Values.conf.drydock.database "database_connect_string" | quote | trunc 0 -}}
 {{- end -}}
 
+# Render MAAS connection string if it is not explicitly configured
+{{- if empty .Values.conf.drydock.maasdriver.maas_api_url -}}
+{{- tuple "maas_region" "internal" "region_api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.drydock.maasdriver "maas_api_url" | quote | trunc 0 -}}
+{{- end -}}
+
 {{- if empty .Values.conf.drydock.keystone_authtoken.auth_uri -}}
 {{- tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.drydock.keystone_authtoken "auth_uri" | quote | trunc 0 -}}
 {{- end -}}

charts/drydock/templates/secret-tls.yaml

@@ -0,0 +1,18 @@
+{{/*
+# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{- if .Values.manifests.secret_tls }}
+{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "physicalprovisioner" ) -}}
+{{- end }}

charts/drydock/values.yaml

@@ -131,6 +131,7 @@ manifests:
   secret_keystone: true
   secret_database: true
   secret_ssh_key: true
+  secret_tls: true
   configmap_etc: true
   configmap_bin: true
   service_drydock: true
@@ -234,7 +235,12 @@ endpoints:
       default: http
     host_fqdn_override:
       default: null
-      public: drydock.test.local
+      public:
+        host: drydock.test.local
+#        tls:
+#          crt: replace
+#          ca: replace
+#          key: replace
   postgresql:
     name: postgresql
     auth:
@@ -254,7 +260,29 @@ endpoints:
         default: 5432
     host_fqdn_override:
       default: null
-
+  maas_region:
+    name: maas-region
+    auth:
+      admin:
+        username: admin
+        password: admin
+        email: none@none
+    hosts:
+      default: maas-region
+      public: maas
+    path:
+      default: /MAAS
+    scheme:
+      default: http
+    port:
+      region_api:
+        default: 80
+        public: 80
+        nodeport: 31900
+      region_proxy:
+        default: 8000
+    host_fqdn_override:
+      default: null
 secrets:
   identity:
     admin: drydock-keystone-admin
@@ -263,6 +291,10 @@ secrets:
     admin: drydock-postgresql-admin
     user: drydock-postgresql-user
   ssh_key: ssh-private-key
+  tls:
+    physicalprovisioner:
+      api:
+        public: drydock-tls-public
 
 # Settings for drydock.conf
 conf:
@@ -288,7 +320,7 @@ conf:
       control_logger_name: '${global_logger_name}.control'
     maasdriver:
       maas_api_key: 'override_this'
-      maas_api_url: 'override_this'
+      maas_api_url: null
     plugins:
       ingester:
         - 'drydock_provisioner.ingester.plugins.yaml.YamlIngester'

python/drydock_provisioner/drivers/node/maasdriver/api_client.py

@@ -54,7 +54,8 @@ class MaasOauth(req_auth.AuthBase):
 
 class MaasRequestFactory(object):
     def __init__(self, base_url, apikey):
-        self.base_url = base_url
+        # The URL in the config should end in /MAAS/, but the api is behind /MAAS/api/2.0/
+        self.base_url = base_url + "/api/2.0/"
         self.apikey = apikey
 
         self.signer = MaasOauth(apikey)

python/drydock_provisioner/ingester/plugins/deckhand.py

@@ -19,13 +19,23 @@ import jsonschema
 import os
 import pkg_resources
 import copy
+import hashlib
 
 import drydock_provisioner.objects.fields as hd_fields
 
+from beaker.cache import CacheManager
+from beaker.util import parse_cache_config_options
+
 from drydock_provisioner import error as errors
 from drydock_provisioner import objects
 from drydock_provisioner.ingester.plugins import IngesterPlugin
 
+cache_opts = {
+    'cache.type': 'memory',
+    'expire': 1800,
+}
+
+cache = CacheManager(**parse_cache_config_options(cache_opts))
 
 class DeckhandIngester(IngesterPlugin):
     def __init__(self):
@@ -44,8 +54,20 @@ class DeckhandIngester(IngesterPlugin):
 
         :returns: a tuple of a status response and a list of parsed objects from drydock_provisioner.objects
         """
+        def local_parse():
+            return self.parse_docs(kwargs.get('content'))
+
         if 'content' in kwargs:
-            parse_status, models = self.parse_docs(kwargs.get('content'))
+            try:
+                # Hash the input to use as the cache key. This is not a security
+                # related hash, so use cheap and fast MD5
+                hv = hashlib.md5(kwargs.get('content', b'')).hexdigest()
+                local_cache = cache.get_cache('parsed_docs')
+                results = local_cache.get(key=hv, createfunc=local_parse)
+                parse_status, models = results
+            except Exception as ex:
+                self.logger.debug("Error parsing design - hash %s", hv, exc_info=ex)
+                raise ex
         else:
             raise ValueError('Missing parameter "content"')
 

python/drydock_provisioner/orchestrator/orchestrator.py

@@ -269,10 +269,10 @@ class Orchestrator(object):
                         site_design,
                         state_manager=self.state_manager,
                         resolve_aliases=resolve_aliases)
-                except Exception:
+                except Exception as ex:
                     node_failed.append(n)
-                    self.logger.error(
+                    self.logger.debug(
-                        "Failed to build applied model for node %s." % n.name)
+                        "Failed to build applied model for node %s.", n.name, exc_info=ex)
             if node_failed:
                 raise errors.DesignError(
                     "Failed to build applied model for %s" % ",".join(

python/drydock_provisioner/statemgmt/design/resolver.py

@@ -28,7 +28,7 @@ from drydock_provisioner.config import config_mgr
 
 cache_opts = {
     'cache.type': 'memory',
-    'expire': 300,
+    'expire': 180,
 }
 
 cache = CacheManager(**parse_cache_config_options(cache_opts))

python/tests/unit/test_api_nodes_unit.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 """Test Nodes API"""
 from falcon import testing
-from unittest.mock import Mock
 
 import pytest
 import json
@@ -85,20 +84,15 @@ class TestNodesApiUnit(object):
 
 
 @pytest.fixture()
-def mock_process_node_filter(deckhand_orchestrator):
+def mock_process_node_filter(mocker, deckhand_orchestrator):
-    def side_effect(**kwargs):
+    n1 = objects.BaremetalNode()
-        n1 = objects.BaremetalNode()
+    n1.name = 'n1'
-        n1.name = 'n1'
+    n1.site = 'test1'
-        n1.site = 'test1'
+    n2 = objects.BaremetalNode()
-        n2 = objects.BaremetalNode()
+    n2.name = 'n2'
-        n2.name = 'n2'
+    n2.site = 'test2'
-        n2.site = 'test2'
+    mock_results = [n1, n2]
-        return [n1, n2]
 
-    deckhand_orchestrator.real_process_node_filter = deckhand_orchestrator.process_node_filter
+    with mocker.patch('drydock_provisioner.orchestrator.orchestrator.Orchestrator.process_node_filter',
-    deckhand_orchestrator.process_node_filter = Mock(side_effect=side_effect)
+                      mocker.MagicMock(return_value=mock_results)):
-
+        yield
-    yield
-    deckhand_orchestrator.process_node_filter = Mock(
-        wraps=None, side_effect=None)
-    deckhand_orchestrator.process_node_filter = deckhand_orchestrator.real_process_node_filter

python/tests/unit/test_api_validation.py

@@ -27,7 +27,7 @@ LOG = logging.getLogger(__name__)
 
 
 class TestValidationApi(object):
-    def test_post_validation_resp(self, input_files, falcontest, drydock_state,
+    def test_post_validation_resp(self, setup_logging, input_files, falcontest, drydock_state,
                                   mock_get_build_data):
 
         input_file = input_files.join("deckhand_fullsite.yaml")
@@ -52,7 +52,7 @@ class TestValidationApi(object):
         LOG.debug(result.text)
         assert result.status == falcon.HTTP_200
 
-    def test_href_error(self, input_files, falcontest):
+    def test_href_error(self, setup_logging, input_files, falcontest):
         url = '/api/v1.0/validatedesign'
         hdr = {
             'Content-Type': 'application/json',
@@ -72,7 +72,7 @@ class TestValidationApi(object):
         LOG.debug(result.text)
         assert result.status == falcon.HTTP_400
 
-    def test_json_data_error(self, input_files, falcontest):
+    def test_json_data_error(self, setup_logging, input_files, falcontest):
         url = '/api/v1.0/validatedesign'
         hdr = {
             'Content-Type': 'application/json',
@@ -88,7 +88,7 @@ class TestValidationApi(object):
         LOG.debug(result.text)
         assert result.status == falcon.HTTP_400
 
-    def test_invalid_post_resp(self, input_files, falcontest, drydock_state,
+    def test_invalid_post_resp(self, setup_logging, input_files, falcontest, drydock_state,
                                mock_get_build_data):
         input_file = input_files.join("invalid_validation.yaml")
         design_ref = "file://%s" % str(input_file)

requirements-host.txt

@@ -1,6 +1,7 @@
 # These are host packages needed for Drydock
 # that don't come on a minimal Ubuntu install
 libvirt-dev
+libyaml-dev
 pkg-config
 python3-dev
 gcc

tools/helm_tk.sh

@@ -18,7 +18,7 @@
 HELM=$1
 HTK_REPO=${HTK_REPO:-"https://github.com/openstack/openstack-helm-infra"}
 HTK_PATH=${HTK_PATH:-""}
-HTK_STABLE_COMMIT=${HTK_COMMIT:-"2fce7e821201a5f578331370703e164d5a932fcc"}
+HTK_STABLE_COMMIT=${HTK_COMMIT:-"42249d4243d892b7d04ce6aed9b0c8d7edcbfc7a"}
 DEP_UP_LIST=${DEP_UP_LIST:-"drydock"}
 BUILD_DIR=${BUILD_DIR:-$(mktemp -d)}