diff --git a/charts/drydock/templates/configmap-etc.yaml b/charts/drydock/templates/configmap-etc.yaml index 95759159..21d5de01 100644 --- a/charts/drydock/templates/configmap-etc.yaml +++ b/charts/drydock/templates/configmap-etc.yaml @@ -22,6 +22,11 @@ {{- tuple "postgresql" "internal" "user" "postgresql" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | set .Values.conf.drydock.database "database_connect_string" | quote | trunc 0 -}} {{- end -}} +# Render MAAS connection string if it is not explicitly configured +{{- if empty .Values.conf.drydock.maasdriver.maas_api_url -}} +{{- tuple "maas_region" "internal" "region_api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.drydock.maasdriver "maas_api_url" | quote | trunc 0 -}} +{{- end -}} + {{- if empty .Values.conf.drydock.keystone_authtoken.auth_uri -}} {{- tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.drydock.keystone_authtoken "auth_uri" | quote | trunc 0 -}} {{- end -}} diff --git a/charts/drydock/templates/secret-tls.yaml b/charts/drydock/templates/secret-tls.yaml new file mode 100644 index 00000000..3a7cace5 --- /dev/null +++ b/charts/drydock/templates/secret-tls.yaml @@ -0,0 +1,18 @@ +{{/* +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- if .Values.manifests.secret_tls }} +{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "physicalprovisioner" ) -}} +{{- end }} diff --git a/charts/drydock/values.yaml b/charts/drydock/values.yaml index 73336572..fc0fa9ab 100644 --- a/charts/drydock/values.yaml +++ b/charts/drydock/values.yaml @@ -131,6 +131,7 @@ manifests: secret_keystone: true secret_database: true secret_ssh_key: true + secret_tls: true configmap_etc: true configmap_bin: true service_drydock: true @@ -234,7 +235,12 @@ endpoints: default: http host_fqdn_override: default: null - public: drydock.test.local + public: + host: drydock.test.local +# tls: +# crt: replace +# ca: replace +# key: replace postgresql: name: postgresql auth: @@ -254,7 +260,29 @@ endpoints: default: 5432 host_fqdn_override: default: null - + maas_region: + name: maas-region + auth: + admin: + username: admin + password: admin + email: none@none + hosts: + default: maas-region + public: maas + path: + default: /MAAS + scheme: + default: http + port: + region_api: + default: 80 + public: 80 + nodeport: 31900 + region_proxy: + default: 8000 + host_fqdn_override: + default: null secrets: identity: admin: drydock-keystone-admin @@ -263,6 +291,10 @@ secrets: admin: drydock-postgresql-admin user: drydock-postgresql-user ssh_key: ssh-private-key + tls: + physicalprovisioner: + api: + public: drydock-tls-public # Settings for drydock.conf conf: @@ -288,7 +320,7 @@ conf: control_logger_name: '${global_logger_name}.control' maasdriver: maas_api_key: 'override_this' - maas_api_url: 'override_this' + maas_api_url: null plugins: ingester: - 'drydock_provisioner.ingester.plugins.yaml.YamlIngester' diff --git a/python/drydock_provisioner/drivers/node/maasdriver/api_client.py b/python/drydock_provisioner/drivers/node/maasdriver/api_client.py index a5c48e3d..8f56dc8b 100644 --- a/python/drydock_provisioner/drivers/node/maasdriver/api_client.py +++ b/python/drydock_provisioner/drivers/node/maasdriver/api_client.py @@ -54,7 +54,8 @@ class MaasOauth(req_auth.AuthBase): class MaasRequestFactory(object): def __init__(self, base_url, apikey): - self.base_url = base_url + # The URL in the config should end in /MAAS/, but the api is behind /MAAS/api/2.0/ + self.base_url = base_url + "/api/2.0/" self.apikey = apikey self.signer = MaasOauth(apikey) diff --git a/python/drydock_provisioner/ingester/plugins/deckhand.py b/python/drydock_provisioner/ingester/plugins/deckhand.py index 7bf650bb..4a8ae033 100644 --- a/python/drydock_provisioner/ingester/plugins/deckhand.py +++ b/python/drydock_provisioner/ingester/plugins/deckhand.py @@ -19,13 +19,23 @@ import jsonschema import os import pkg_resources import copy +import hashlib import drydock_provisioner.objects.fields as hd_fields +from beaker.cache import CacheManager +from beaker.util import parse_cache_config_options + from drydock_provisioner import error as errors from drydock_provisioner import objects from drydock_provisioner.ingester.plugins import IngesterPlugin +cache_opts = { + 'cache.type': 'memory', + 'expire': 1800, +} + +cache = CacheManager(**parse_cache_config_options(cache_opts)) class DeckhandIngester(IngesterPlugin): def __init__(self): @@ -44,8 +54,20 @@ class DeckhandIngester(IngesterPlugin): :returns: a tuple of a status response and a list of parsed objects from drydock_provisioner.objects """ + def local_parse(): + return self.parse_docs(kwargs.get('content')) + if 'content' in kwargs: - parse_status, models = self.parse_docs(kwargs.get('content')) + try: + # Hash the input to use as the cache key. This is not a security + # related hash, so use cheap and fast MD5 + hv = hashlib.md5(kwargs.get('content', b'')).hexdigest() + local_cache = cache.get_cache('parsed_docs') + results = local_cache.get(key=hv, createfunc=local_parse) + parse_status, models = results + except Exception as ex: + self.logger.debug("Error parsing design - hash %s", hv, exc_info=ex) + raise ex else: raise ValueError('Missing parameter "content"') diff --git a/python/drydock_provisioner/orchestrator/orchestrator.py b/python/drydock_provisioner/orchestrator/orchestrator.py index 473d1b82..863b1670 100644 --- a/python/drydock_provisioner/orchestrator/orchestrator.py +++ b/python/drydock_provisioner/orchestrator/orchestrator.py @@ -269,10 +269,10 @@ class Orchestrator(object): site_design, state_manager=self.state_manager, resolve_aliases=resolve_aliases) - except Exception: + except Exception as ex: node_failed.append(n) - self.logger.error( - "Failed to build applied model for node %s." % n.name) + self.logger.debug( + "Failed to build applied model for node %s.", n.name, exc_info=ex) if node_failed: raise errors.DesignError( "Failed to build applied model for %s" % ",".join( diff --git a/python/drydock_provisioner/statemgmt/design/resolver.py b/python/drydock_provisioner/statemgmt/design/resolver.py index 639cdba9..0feaeabd 100644 --- a/python/drydock_provisioner/statemgmt/design/resolver.py +++ b/python/drydock_provisioner/statemgmt/design/resolver.py @@ -28,7 +28,7 @@ from drydock_provisioner.config import config_mgr cache_opts = { 'cache.type': 'memory', - 'expire': 300, + 'expire': 180, } cache = CacheManager(**parse_cache_config_options(cache_opts)) diff --git a/python/tests/unit/test_api_nodes_unit.py b/python/tests/unit/test_api_nodes_unit.py index 20ddbe13..76edf8ef 100644 --- a/python/tests/unit/test_api_nodes_unit.py +++ b/python/tests/unit/test_api_nodes_unit.py @@ -13,7 +13,6 @@ # limitations under the License. """Test Nodes API""" from falcon import testing -from unittest.mock import Mock import pytest import json @@ -85,20 +84,15 @@ class TestNodesApiUnit(object): @pytest.fixture() -def mock_process_node_filter(deckhand_orchestrator): - def side_effect(**kwargs): - n1 = objects.BaremetalNode() - n1.name = 'n1' - n1.site = 'test1' - n2 = objects.BaremetalNode() - n2.name = 'n2' - n2.site = 'test2' - return [n1, n2] +def mock_process_node_filter(mocker, deckhand_orchestrator): + n1 = objects.BaremetalNode() + n1.name = 'n1' + n1.site = 'test1' + n2 = objects.BaremetalNode() + n2.name = 'n2' + n2.site = 'test2' + mock_results = [n1, n2] - deckhand_orchestrator.real_process_node_filter = deckhand_orchestrator.process_node_filter - deckhand_orchestrator.process_node_filter = Mock(side_effect=side_effect) - - yield - deckhand_orchestrator.process_node_filter = Mock( - wraps=None, side_effect=None) - deckhand_orchestrator.process_node_filter = deckhand_orchestrator.real_process_node_filter + with mocker.patch('drydock_provisioner.orchestrator.orchestrator.Orchestrator.process_node_filter', + mocker.MagicMock(return_value=mock_results)): + yield diff --git a/python/tests/unit/test_api_validation.py b/python/tests/unit/test_api_validation.py index c04fee0d..f64d95eb 100644 --- a/python/tests/unit/test_api_validation.py +++ b/python/tests/unit/test_api_validation.py @@ -27,7 +27,7 @@ LOG = logging.getLogger(__name__) class TestValidationApi(object): - def test_post_validation_resp(self, input_files, falcontest, drydock_state, + def test_post_validation_resp(self, setup_logging, input_files, falcontest, drydock_state, mock_get_build_data): input_file = input_files.join("deckhand_fullsite.yaml") @@ -52,7 +52,7 @@ class TestValidationApi(object): LOG.debug(result.text) assert result.status == falcon.HTTP_200 - def test_href_error(self, input_files, falcontest): + def test_href_error(self, setup_logging, input_files, falcontest): url = '/api/v1.0/validatedesign' hdr = { 'Content-Type': 'application/json', @@ -72,7 +72,7 @@ class TestValidationApi(object): LOG.debug(result.text) assert result.status == falcon.HTTP_400 - def test_json_data_error(self, input_files, falcontest): + def test_json_data_error(self, setup_logging, input_files, falcontest): url = '/api/v1.0/validatedesign' hdr = { 'Content-Type': 'application/json', @@ -88,7 +88,7 @@ class TestValidationApi(object): LOG.debug(result.text) assert result.status == falcon.HTTP_400 - def test_invalid_post_resp(self, input_files, falcontest, drydock_state, + def test_invalid_post_resp(self, setup_logging, input_files, falcontest, drydock_state, mock_get_build_data): input_file = input_files.join("invalid_validation.yaml") design_ref = "file://%s" % str(input_file) diff --git a/requirements-host.txt b/requirements-host.txt index ee59c621..39087e55 100644 --- a/requirements-host.txt +++ b/requirements-host.txt @@ -1,6 +1,7 @@ # These are host packages needed for Drydock # that don't come on a minimal Ubuntu install libvirt-dev +libyaml-dev pkg-config python3-dev gcc diff --git a/tools/helm_tk.sh b/tools/helm_tk.sh index 1a1a366b..b8d2fe00 100755 --- a/tools/helm_tk.sh +++ b/tools/helm_tk.sh @@ -18,7 +18,7 @@ HELM=$1 HTK_REPO=${HTK_REPO:-"https://github.com/openstack/openstack-helm-infra"} HTK_PATH=${HTK_PATH:-""} -HTK_STABLE_COMMIT=${HTK_COMMIT:-"2fce7e821201a5f578331370703e164d5a932fcc"} +HTK_STABLE_COMMIT=${HTK_COMMIT:-"42249d4243d892b7d04ce6aed9b0c8d7edcbfc7a"} DEP_UP_LIST=${DEP_UP_LIST:-"drydock"} BUILD_DIR=${BUILD_DIR:-$(mktemp -d)}