From 05cc8f9d77aef9dfacad2932534fc3edb151f1dc Mon Sep 17 00:00:00 2001 From: Angie Wang Date: Fri, 22 Mar 2019 16:35:01 -0400 Subject: [PATCH] Move the image specification to the armada manifest This commit updates to move the image specification from overrides generation to armada manifest and get the download image list from both overrides files and armada manifest. Story: 2005350 Task: 30312 Change-Id: Iea17a516c53cce4a0baaef684aa3c6cf6b3eff28 Signed-off-by: Angie Wang --- .../manifests/manifest.yaml | 196 +++++++++++++++++- .../helm-charts/rbd-provisioner/values.yaml | 2 +- sysinv/sysinv/sysinv/requirements.txt | 1 + sysinv/sysinv/sysinv/sysinv/cmd/helm.py | 18 +- .../sysinv/sysinv/conductor/kube_app.py | 155 +++++++++----- sysinv/sysinv/sysinv/sysinv/helm/aodh.py | 21 -- sysinv/sysinv/sysinv/sysinv/helm/barbican.py | 18 -- sysinv/sysinv/sysinv/sysinv/helm/base.py | 21 -- .../sysinv/sysinv/sysinv/helm/ceilometer.py | 22 -- .../sysinv/sysinv/helm/ceph_pools_audit.py | 8 - sysinv/sysinv/sysinv/sysinv/helm/cinder.py | 31 --- sysinv/sysinv/sysinv/sysinv/helm/common.py | 48 ----- sysinv/sysinv/sysinv/sysinv/helm/garbd.py | 9 - sysinv/sysinv/sysinv/sysinv/helm/glance.py | 25 --- sysinv/sysinv/sysinv/sysinv/helm/gnocchi.py | 19 -- sysinv/sysinv/sysinv/sysinv/helm/heat.py | 20 -- sysinv/sysinv/sysinv/sysinv/helm/helm.py | 16 +- sysinv/sysinv/sysinv/sysinv/helm/horizon.py | 13 -- sysinv/sysinv/sysinv/sysinv/helm/ironic.py | 22 -- sysinv/sysinv/sysinv/sysinv/helm/keystone.py | 22 -- sysinv/sysinv/sysinv/sysinv/helm/libvirt.py | 12 -- sysinv/sysinv/sysinv/sysinv/helm/magnum.py | 18 -- sysinv/sysinv/sysinv/sysinv/helm/mariadb.py | 11 - sysinv/sysinv/sysinv/sysinv/helm/neutron.py | 29 +-- sysinv/sysinv/sysinv/sysinv/helm/nova.py | 39 ---- .../sysinv/sysinv/helm/nova_api_proxy.py | 12 -- sysinv/sysinv/sysinv/sysinv/helm/panko.py | 18 -- sysinv/sysinv/sysinv/sysinv/helm/rabbitmq.py | 11 - .../sysinv/sysinv/helm/rbd_provisioner.py | 16 -- 29 files changed, 301 insertions(+), 552 deletions(-) diff --git a/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml b/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml index 3485c562ab..f8b260e7fd 100644 --- a/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml +++ b/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml @@ -126,6 +126,11 @@ data: - type: job labels: app: rbd-provisioner + values: + images: + tags: + # TODO: Remove after ceph upgrade + rbd_provisioner_storage_init: docker.io/starlingx/stx-ceph-config-helper:master-centos-dev-latest source: type: tar location: http://172.17.0.1/helm_charts/rbd-provisioner-0.1.0.tgz @@ -187,7 +192,7 @@ data: values: monitoring: prometheus: - enabled: true + enabled: false labels: server: node_selector_key: openstack-control-plane @@ -195,6 +200,9 @@ data: prometheus_mysql_exporter: node_selector_key: openstack-control-plane node_selector_value: enabled + images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/starlingx/stx-heat:master-centos-dev-latest pod: affinity: anti: @@ -233,6 +241,9 @@ data: server: node_selector_key: openstack-compute-node node_selector_value: enabled + images: + tags: + garbd: docker.io/starlingx/stx-mariadb:master-centos-stable-latest source: type: tar location: http://172.17.0.1/helm_charts/garbd-0.1.0.tgz @@ -271,7 +282,7 @@ data: node_selector_value: enabled monitoring: prometheus: - enabled: true + enabled: false source: type: tar location: http://172.17.0.1/helm_charts/memcached-0.1.0.tgz @@ -310,7 +321,7 @@ data: values: monitoring: prometheus: - enabled: true + enabled: false labels: server: node_selector_key: openstack-control-plane @@ -318,6 +329,9 @@ data: prometheus_rabbitmq_exporter: node_selector_key: openstack-control-plane node_selector_value: enabled + images: + tags: + prometheus_rabbitmq_exporter_helm_tests: docker.io/starlingx/stx-heat:master-centos-dev-latest pod: affinity: anti: @@ -376,7 +390,23 @@ data: job: node_selector_key: openstack-control-plane node_selector_value: enabled + images: + tags: + bootstrap: docker.io/starlingx/stx-heat:master-centos-dev-latest + db_drop: docker.io/starlingx/stx-heat:master-centos-dev-latest + db_init: docker.io/starlingx/stx-heat:master-centos-dev-latest + keystone_api: docker.io/starlingx/stx-keystone:master-centos-dev-latest + keystone_credential_rotate: docker.io/starlingx/stx-keystone:master-centos-dev-latest + keystone_credential_setup: docker.io/starlingx/stx-keystone:master-centos-dev-latest + keystone_db_sync: docker.io/starlingx/stx-keystone:master-centos-dev-latest + keystone_domain_manage: docker.io/starlingx/stx-keystone:master-centos-dev-latest + keystone_fernet_rotate: docker.io/starlingx/stx-keystone:master-centos-dev-latest + keystone_fernet_setup: docker.io/starlingx/stx-keystone:master-centos-dev-latest + ks_user: docker.io/starlingx/stx-heat:master-centos-dev-latest pod: + user: + keystone: + uid: 0 replicas: api: 2 affinity: @@ -426,6 +456,17 @@ data: job: node_selector_key: openstack-control-plane node_selector_value: enabled + images: + tags: + barbican_api: docker.io/starlingx/stx-barbican:master-centos-dev-latest + barbican_db_sync: docker.io/starlingx/stx-barbican:master-centos-dev-latest + bootstrap: docker.io/starlingx/stx-heat:master-centos-dev-latest + db_drop: docker.io/starlingx/stx-heat:master-centos-dev-latest + db_init: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_endpoints: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_service: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_user: docker.io/starlingx/stx-heat:master-centos-dev-latest + scripted_test: docker.io/starlingx/stx-heat:master-centos-dev-latest pod: replicas: api: 2 @@ -479,6 +520,18 @@ data: registry: node_selector_key: openstack-control-plane node_selector_value: enabled + images: + tags: + bootstrap: docker.io/starlingx/stx-heat:master-centos-dev-latest + db_drop: docker.io/starlingx/stx-heat:master-centos-dev-latest + db_init: docker.io/starlingx/stx-heat:master-centos-dev-latest + glance_storage_init: docker.io/starlingx/stx-ceph-config-helper:master-centos-dev-latest + glance_api: docker.io/starlingx/stx-glance:master-centos-dev-latest + glance_db_sync: docker.io/starlingx/stx-glance:master-centos-dev-latest + glance_registry: docker.io/starlingx/stx-glance:master-centos-dev-latest + ks_endpoints: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_service: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_user: docker.io/starlingx/stx-heat:master-centos-dev-latest pod: replicas: api: 2 @@ -539,6 +592,22 @@ data: volume: node_selector_key: openstack-control-plane node_selector_value: enabled + images: + tags: + bootstrap: docker.io/starlingx/stx-heat:master-centos-dev-latest + cinder_api: docker.io/starlingx/stx-cinder:master-centos-dev-latest + cinder_backup: docker.io/starlingx/stx-cinder:master-centos-dev-latest + cinder_backup_storage_init: docker.io/starlingx/stx-ceph-config-helper:master-centos-dev-latest + cinder_db_sync: docker.io/starlingx/stx-cinder:master-centos-dev-latest + cinder_scheduler: docker.io/starlingx/stx-cinder:master-centos-dev-latest + cinder_storage_init: docker.io/starlingx/stx-ceph-config-helper:master-centos-dev-latest + cinder_volume: docker.io/starlingx/stx-cinder:master-centos-dev-latest + cinder_volume_usage_audit: docker.io/starlingx/stx-cinder:master-centos-dev-latest + db_drop: docker.io/starlingx/stx-heat:master-centos-dev-latest + db_init: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_endpoints: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_service: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_user: docker.io/starlingx/stx-heat:master-centos-dev-latest pod: replicas: api: 2 @@ -580,6 +649,9 @@ data: libvirt: node_selector_key: openstack-compute-node node_selector_value: enabled + images: + tags: + libvirt: docker.io/starlingx/stx-libvirt:master-centos-stable-latest source: type: tar location: http://172.17.0.1/helm_charts/libvirt-0.1.0.tgz @@ -717,6 +789,29 @@ data: test: node_selector_key: openstack-control-plane node_selector_value: enabled + images: + tags: + bootstrap: docker.io/starlingx/stx-heat:master-centos-dev-latest + db_drop: docker.io/starlingx/stx-heat:master-centos-dev-latest + db_init: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_user: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_service: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_endpoints: docker.io/starlingx/stx-heat:master-centos-dev-latest + nova_api: docker.io/starlingx/stx-nova:master-centos-dev-latest + nova_cell_setup: docker.io/starlingx/stx-nova:master-centos-dev-latest + nova_cell_setup_init: docker.io/starlingx/stx-heat:master-centos-dev-latest + nova_compute: docker.io/starlingx/stx-nova:master-centos-dev-latest + nova_compute_ironic: docker.io/starlingx/stx-nova:master-centos-dev-latest + nova_compute_ssh: docker.io/starlingx/stx-nova:master-centos-dev-latest + nova_conductor: docker.io/starlingx/stx-nova:master-centos-dev-latest + nova_consoleauth: docker.io/starlingx/stx-nova:master-centos-dev-latest + nova_db_sync: docker.io/starlingx/stx-nova:master-centos-dev-latest + nova_novncproxy: docker.io/starlingx/stx-nova:master-centos-dev-latest + nova_placement: docker.io/starlingx/stx-nova:master-centos-dev-latest + nova_scheduler: docker.io/starlingx/stx-nova:master-centos-dev-latest + nova_spiceproxy: docker.io/starlingx/stx-nova:master-centos-dev-latest + nova_spiceproxy_assets: docker.io/starlingx/stx-nova:master-centos-dev-latest + nova_storage_init: docker.io/starlingx/stx-ceph-config-helper:master-centos-dev-latest pod: replicas: api_metadata: 1 @@ -859,6 +954,11 @@ data: labels: release_group: osh-openstack-nova-api-proxy component: test + values: + images: + tags: + nova_api_proxy: docker.io/starlingx/stx-nova-api-proxy:master-centos-dev-latest + ks_endpoints: docker.io/starlingx/stx-heat:master-centos-dev-latest source: type: tar location: http://172.17.0.1/helm_charts/nova-api-proxy-0.1.0.tgz @@ -944,6 +1044,23 @@ data: test: node_selector_key: openstack-control-plane node_selector_value: enabled + images: + tags: + bootstrap: docker.io/starlingx/stx-heat:master-centos-dev-latest + db_init: docker.io/starlingx/stx-heat:master-centos-dev-latest + neutron_db_sync: docker.io/starlingx/stx-neutron:master-centos-dev-latest + db_drop: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_user: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_service: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_endpoints: docker.io/starlingx/stx-heat:master-centos-dev-latest + neutron_server: docker.io/starlingx/stx-neutron:master-centos-dev-latest + neutron_dhcp: docker.io/starlingx/stx-neutron:master-centos-dev-latest + neutron_metadata: docker.io/starlingx/stx-neutron:master-centos-dev-latest + neutron_l3: docker.io/starlingx/stx-neutron:master-centos-dev-latest + neutron_openvswitch_agent: docker.io/starlingx/stx-neutron:master-centos-dev-latest + neutron_linuxbridge_agent: docker.io/starlingx/stx-neutron:master-centos-dev-latest + neutron_sriov_agent: docker.io/starlingx/stx-neutron:master-centos-dev-latest + neutron_sriov_agent_init: docker.io/starlingx/stx-neutron:master-centos-dev-latest network: interface: tunnel: docker0 @@ -1081,6 +1198,21 @@ data: job: node_selector_key: openstack-control-plane node_selector_value: enabled + images: + tags: + bootstrap: docker.io/starlingx/stx-heat:master-centos-dev-latest + db_drop: docker.io/starlingx/stx-heat:master-centos-dev-latest + db_init: docker.io/starlingx/stx-heat:master-centos-dev-latest + heat_api: docker.io/starlingx/stx-heat:master-centos-dev-latest + heat_cfn: docker.io/starlingx/stx-heat:master-centos-dev-latest + heat_cloudwatch: docker.io/starlingx/stx-heat:master-centos-dev-latest + heat_db_sync: docker.io/starlingx/stx-heat:master-centos-dev-latest + heat_engine: docker.io/starlingx/stx-heat:master-centos-dev-latest + heat_engine_cleaner: docker.io/starlingx/stx-heat:master-centos-dev-latest + heat_purge_deleted: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_endpoints: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_service: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_user: docker.io/starlingx/stx-heat:master-centos-dev-latest conf: policy: stacks:global_index: rule:context_is_admin @@ -1131,6 +1263,20 @@ data: release_group: osh-openstack-aodh component: test values: + images: + tags: + bootstrap: docker.io/starlingx/stx-heat:master-centos-dev-latest + db_drop: docker.io/starlingx/stx-heat:master-centos-dev-latest + db_init: docker.io/starlingx/stx-heat:master-centos-dev-latest + aodh_api: docker.io/starlingx/stx-aodh:master-centos-dev-latest + aodh_alarms_cleaner: docker.io/starlingx/stx-aodh:master-centos-dev-latest + aodh_db_sync: docker.io/starlingx/stx-aodh:master-centos-dev-latest + aodh_evaluator: docker.io/starlingx/stx-aodh:master-centos-dev-latest + aodh_listener: docker.io/starlingx/stx-aodh:master-centos-dev-latest + aodh_notifier: docker.io/starlingx/stx-aodh:master-centos-dev-latest + ks_endpoints: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_service: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_user: docker.io/starlingx/stx-heat:master-centos-dev-latest pod: user: aodh: @@ -1179,6 +1325,18 @@ data: release_group: osh-openstack-gnocchi component: test values: + images: + tags: + db_init: docker.io/starlingx/stx-gnocchi:master-centos-dev-latest + db_init_indexer: docker.io/starlingx/stx-gnocchi:master-centos-dev-latest + db_sync: docker.io/starlingx/stx-gnocchi:master-centos-dev-latest + gnocchi_api: docker.io/starlingx/stx-gnocchi:master-centos-dev-latest + gnocchi_metricd: docker.io/starlingx/stx-gnocchi:master-centos-dev-latest + gnocchi_resources_cleaner: docker.io/starlingx/stx-gnocchi:master-centos-dev-latest + gnocchi_statsd: docker.io/starlingx/stx-gnocchi:master-centos-dev-latest + ks_endpoints: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_service: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_user: docker.io/starlingx/stx-heat:master-centos-dev-latest conf: gnocchi: indexer: @@ -1324,6 +1482,17 @@ data: anti: type: default: requiredDuringSchedulingIgnoredDuringExecution + images: + tags: + bootstrap: docker.io/starlingx/stx-heat:master-centos-dev-latest + db_drop: docker.io/starlingx/stx-heat:master-centos-dev-latest + db_init: docker.io/starlingx/stx-heat:master-centos-dev-latest + panko_db_sync: docker.io/starlingx/stx-panko:master-centos-dev-latest + panko_api: docker.io/starlingx/stx-panko:master-centos-dev-latest + panko_events_cleaner: docker.io/starlingx/stx-panko:master-centos-dev-latest + ks_endpoints: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_service: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_user: docker.io/starlingx/stx-heat:master-centos-dev-latest jobs: events_cleaner: # hourly at the 10 minute mark @@ -1364,6 +1533,21 @@ data: release_group: osh-openstack-ceilometer component: test values: + images: + tags: + ks_service: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_user: docker.io/starlingx/stx-heat:master-centos-dev-latest + ks_endpoints: docker.io/starlingx/stx-heat:master-centos-dev-latest + db_init: docker.io/starlingx/stx-ceilometer:master-centos-dev-latest + db_init_mongodb: docker.io/starlingx/stx-ceilometer:master-centos-dev-latest + ceilometer_db_sync: docker.io/starlingx/stx-ceilometer:master-centos-dev-latest + ceilometer_central: docker.io/starlingx/stx-ceilometer:master-centos-dev-latest + ceilometer_compute: docker.io/starlingx/stx-ceilometer:master-centos-dev-latest + ceilometer_ipmi: docker.io/starlingx/stx-ceilometer:master-centos-dev-latest + ceilometer_notification: docker.io/starlingx/stx-ceilometer:master-centos-dev-latest + ceilometer_collector: docker.io/starlingx/stx-ceilometer:master-centos-dev-latest + ceilometer_api: docker.io/starlingx/stx-ceilometer:master-centos-dev-latest + test: docker.io/starlingx/stx-ceilometer:master-centos-dev-latest dependencies: static: central: @@ -1871,6 +2055,12 @@ data: labels: release_group: osh-openstack-horizon values: + images: + tags: + db_drop: docker.io/starlingx/stx-heat:master-centos-dev-latest + db_init: docker.io/starlingx/stx-heat:master-centos-dev-latest + horizon: docker.io/starlingx/stx-horizon:master-centos-dev-latest + horizon_db_sync: docker.io/starlingx/stx-horizon:master-centos-dev-latest network: node_port: enabled: 'true' diff --git a/kubernetes/helm-charts/rbd-provisioner/values.yaml b/kubernetes/helm-charts/rbd-provisioner/values.yaml index 17873fbef6..b056e66326 100644 --- a/kubernetes/helm-charts/rbd-provisioner/values.yaml +++ b/kubernetes/helm-charts/rbd-provisioner/values.yaml @@ -151,7 +151,7 @@ ephemeral_pools: # images: tags: - rbd_provisioner: quay.io/external_storage/rbd-provisioner:latest + rbd_provisioner: quay.io/external_storage/rbd-provisioner:v2.1.1-k8s1.11 rbd_provisioner_storage_init: docker.io/port/ceph-config-helper:v1.10.3 pull_policy: "IfNotPresent" local_registry: diff --git a/sysinv/sysinv/sysinv/requirements.txt b/sysinv/sysinv/sysinv/requirements.txt index 11c35fff9d..eec3bfd240 100644 --- a/sysinv/sysinv/sysinv/requirements.txt +++ b/sysinv/sysinv/sysinv/requirements.txt @@ -37,5 +37,6 @@ python-magnumclient>=2.0.0 # Apache-2.0 psutil simplejson>=2.2.0 # MIT rpm +ruamel.yaml>=0.13.14 # MIT docker # Apache-2.0 kubernetes # Apache-2.0 diff --git a/sysinv/sysinv/sysinv/sysinv/cmd/helm.py b/sysinv/sysinv/sysinv/sysinv/cmd/helm.py index e16a2951fa..43bb888c71 100644 --- a/sysinv/sysinv/sysinv/sysinv/cmd/helm.py +++ b/sysinv/sysinv/sysinv/sysinv/cmd/helm.py @@ -20,22 +20,22 @@ from sysinv.helm import helm CONF = cfg.CONF -def create_app_overrides_action(path, app_name=None, repository=None, namespace=None): +def create_app_overrides_action(path, app_name=None, namespace=None): dbapi = api.get_instance() - operator = helm.HelmOperator(dbapi=dbapi, path=path, docker_repository=repository) + operator = helm.HelmOperator(dbapi=dbapi, path=path) operator.generate_helm_application_overrides(app_name, namespace) -def create_armada_app_overrides_action(path, app_name=None, repository=None, namespace=None): +def create_armada_app_overrides_action(path, app_name=None, namespace=None): dbapi = api.get_instance() - operator = helm.HelmOperator(dbapi=dbapi, path=path, docker_repository=repository) + operator = helm.HelmOperator(dbapi=dbapi, path=path) operator.generate_helm_application_overrides(app_name, namespace, armada_format=True) -def create_chart_override_action(path, chart_name=None, repository=None, namespace=None): +def create_chart_override_action(path, chart_name=None, namespace=None): dbapi = api.get_instance() - operator = helm.HelmOperator(dbapi=dbapi, path=path, docker_repository=repository) + operator = helm.HelmOperator(dbapi=dbapi, path=path) operator.generate_helm_chart_overrides(chart_name, namespace) @@ -44,21 +44,18 @@ def add_action_parsers(subparsers): parser.set_defaults(func=create_app_overrides_action) parser.add_argument('path', nargs='?') parser.add_argument('app_name', nargs='?') - parser.add_argument('repository', nargs='?') parser.add_argument('namespace', nargs='?') parser = subparsers.add_parser('create-armada-app-overrides') parser.set_defaults(func=create_armada_app_overrides_action) parser.add_argument('path', nargs='?') parser.add_argument('app_name', nargs='?') - parser.add_argument('repository', nargs='?') parser.add_argument('namespace', nargs='?') parser = subparsers.add_parser('create-chart-overrides') parser.set_defaults(func=create_chart_override_action) parser.add_argument('path', nargs='?') parser.add_argument('chart_name', nargs='?') - parser.add_argument('repository', nargs='?') parser.add_argument('namespace', nargs='?') @@ -75,18 +72,15 @@ def main(): CONF.action.func(CONF.action.path, CONF.action.app_name, - CONF.action.repository, CONF.action.namespace) elif CONF.action.name == 'create-armada-app-overrides': CONF.action.func(CONF.action.path, CONF.action.app_name, - CONF.action.repository, CONF.action.namespace) elif CONF.action.name == 'create-chart-overrides': try: CONF.action.func(CONF.action.path, CONF.action.chart_name, - CONF.action.repository, CONF.action.namespace) except Exception as e: print(e) diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py index 6f33284fb3..1aa9cd240d 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py @@ -16,11 +16,11 @@ import keyring import os import pwd import re +import ruamel.yaml as yaml import shutil import subprocess import threading import time -import yaml from collections import namedtuple from eventlet import greenpool @@ -62,6 +62,7 @@ TARFILE_TRANSFER_CHUNK_SIZE = 1024 * 512 DOCKER_REGISTRY_USER = 'admin' DOCKER_REGISTRY_SERVICE = 'CGCS' DOCKER_REGISTRY_SECRET = 'default-registry-key' +DOCKER_REGISTRY_PORT = '9001' # Helper functions @@ -311,17 +312,18 @@ class AppOperator(object): for r, f in cutils.get_files_matching(path, 'values.yaml'): with open(os.path.join(r, f), 'r') as value_f: try: - y = yaml.load(value_f) + y = yaml.safe_load(value_f) ids = y["images"]["tags"].values() except (TypeError, KeyError): pass image_tags.extend(ids) return list(set(image_tags)) - def _get_image_tags_by_charts(self, app_images_file, charts): + def _get_image_tags_by_charts(self, app_images_file, app_manifest_file): """ Mine the image tags for charts from the images file. Add the - converted image tags to the overrides if the image tags from - the charts do not exist. Intended for system app. + image tags to the manifest file if the image tags from the charts + do not exist in both overrides file and manifest file. Convert + the image tags in the manifest file. Intended for system app. The image tagging conversion(local docker registry address prepended): ${LOCAL_DOCKER_REGISTRY_IP}:${REGISTRY_PORT}/ @@ -329,64 +331,102 @@ class AppOperator(object): """ local_registry_server = self._docker.get_local_docker_registry_server() + manifest_image_tags_updated = False image_tags = [] + + if os.path.exists(app_images_file): + with open(app_images_file, 'r') as f: + images_file = yaml.safe_load(f) + + if os.path.exists(app_manifest_file): + with open(app_manifest_file, 'r') as f: + charts = list(yaml.load_all(f, Loader=yaml.RoundTripLoader)) + for chart in charts: images_charts = {} images_overrides = {} - overrides = chart.namespace + '-' + chart.name + '.yaml' - overrides_file = os.path.join(common.HELM_OVERRIDES_PATH, - overrides) + images_manifest = {} - # Get the image tags by chart from the images file - if os.path.exists(app_images_file): - with open(app_images_file, 'r') as f: + overrides_image_tags_updated = False + chart_image_tags_updated = False + + if "armada/Chart/" in chart['schema']: + chart_data = chart['data'] + chart_name = chart_data['chart_name'] + chart_namespace = chart_data['namespace'] + + # Get the image tags by chart from the images file + if chart_name in images_file: + images_charts = images_file[chart_name] + + # Get the image tags from the overrides file + overrides = chart_namespace + '-' + chart_name + '.yaml' + app_overrides_file = os.path.join(common.HELM_OVERRIDES_PATH, overrides) + if os.path.exists(app_overrides_file): try: - doc = yaml.load(f) - images_charts = doc[chart.name] + with open(app_overrides_file, 'r') as f: + overrides_file = yaml.safe_load(f) + images_overrides = overrides_file['data']['values']['images']['tags'] except (TypeError, KeyError): pass - # Get the image tags from the overrides file - if os.path.exists(overrides_file): - with open(overrides_file, 'r') as f: - try: - y = yaml.load(f) - images_overrides = y["data"]["values"]["images"]["tags"] - except (TypeError, KeyError): - LOG.info("Overrides file %s has no img tags" % - overrides_file) - pass + # Get the image tags from the armada manifest file + try: + images_manifest = chart_data['values']['images']['tags'] + except (TypeError, KeyError): + LOG.info("Armada manifest file has no img tags for " + "chart %s" % chart_name) + pass - # Add the converted image tags to the overrides if the images from - # the chart path do not exist in the overrides - tags_updated = False - for key, image_tag in images_charts.items(): - if (key not in images_overrides and - not image_tag.startswith(local_registry_server)): - images_overrides.update( - {key: '{}/{}'.format(local_registry_server, image_tag)}) - tags_updated = True + # For the image tags from the chart path which do not exist + # in the overrides and manifest file, add to manifest file. + # Convert the image tags in the overrides and manifest file + # with local docker registry address. + # Append the required images to the image_tags list. + for key in images_charts: + if key not in images_overrides: + if key not in images_manifest: + images_manifest.update({key: images_charts[key]}) + if not re.match(r'^.+:.+/', images_manifest[key]): + images_manifest.update( + {key: '{}/{}'.format(local_registry_server, images_manifest[key])}) + chart_image_tags_updated = True + image_tags.append(images_manifest[key]) + else: + if not re.match(r'^.+:.+/', images_overrides[key]): + images_overrides.update( + {key: '{}/{}'.format(local_registry_server, images_overrides[key])}) + overrides_image_tags_updated = True + image_tags.append(images_overrides[key]) - if tags_updated: - with open(overrides_file, 'w') as f: - try: - if "images" not in y["data"]["values"]: - f.seek(0) - f.truncate() - y["data"]["values"]["images"] = {"tags": images_overrides} - else: - y["data"]["values"]["images"]["tags"] = images_overrides + if overrides_image_tags_updated: + with open(app_overrides_file, 'w') as f: + try: + overrides_file["data"]["values"]["images"] = {"tags": images_overrides} + yaml.safe_dump(overrides_file, f, default_flow_style=False) + LOG.info("Overrides file %s updated with new image tags" % + app_overrides_file) + except (TypeError, KeyError): + LOG.error("Overrides file %s fails to update" % + app_overrides_file) - yaml.safe_dump(y, f, explicit_start=True, - default_flow_style=False) - LOG.info("Overrides file %s updated with new image tags" % - overrides_file) - except (TypeError, KeyError): - LOG.error("Overrides file %s fails to update" % - overrides_file) + if chart_image_tags_updated: + if 'values' in chart_data: + chart_data['values']['images'] = {'tags': images_manifest} + else: + chart_data["values"] = {"images": {"tags": images_manifest}} + manifest_image_tags_updated = True - if images_overrides: - image_tags.extend(images_overrides.values()) + if manifest_image_tags_updated: + with open(app_manifest_file, 'w') as f: + try: + yaml.dump_all(charts, f, Dumper=yaml.RoundTripDumper, + explicit_start=True, default_flow_style=False) + LOG.info("Manifest file %s updated with new image tags" % + app_manifest_file) + except Exception as e: + LOG.error("Manifest file %s fails to update with " + "new image tags: %s" % (app_manifest_file, e)) return list(set(image_tags)) @@ -418,7 +458,7 @@ class AppOperator(object): self._save_images_list_by_charts(app) # Get the list of images from the updated images overrides images_to_download = self._get_image_tags_by_charts( - app.imgfile_abs, app.charts) + app.imgfile_abs, app.armada_mfile_abs) else: # For custom apps, mine image tags from application path images_to_download = self._get_image_tags_by_path(app.path) @@ -447,7 +487,7 @@ class AppOperator(object): if os.path.exists(chart_path): with open(chart_path, 'r') as f: try: - y = yaml.load(f) + y = yaml.safe_load(f) images = y["images"]["tags"] except (TypeError, KeyError): LOG.warn("Chart %s has no image tags" % chart_name) @@ -460,7 +500,7 @@ class AppOperator(object): def _retrieve_images_list(self, app_images_file): with open(app_images_file, 'rb') as f: - images_list = yaml.load(f) + images_list = yaml.safe_load(f) return images_list def _download_images(self, app): @@ -473,7 +513,8 @@ class AppOperator(object): # saved images list. saved_images_list = self._retrieve_images_list(app.imgfile_abs) saved_download_images_list = list(saved_images_list.get("download_images")) - images_to_download = self._get_image_tags_by_charts(app.imgfile_abs, app.charts) + images_to_download = self._get_image_tags_by_charts( + app.imgfile_abs, app.armada_mfile_abs) if set(saved_download_images_list) != set(images_to_download): saved_images_list.update({"download_images": images_to_download}) with open(app.imgfile_abs, 'wb') as f: @@ -621,7 +662,7 @@ class AppOperator(object): if os.path.exists(lfile) and os.path.getsize(lfile) > 0: with open(lfile, 'r') as f: try: - y = yaml.load(f) + y = yaml.safe_load(f) labels = y['labels'] except KeyError: raise exception.KubeAppUploadFailure( @@ -795,7 +836,7 @@ class AppOperator(object): def _get_list_of_charts(self, manifest_file): charts = [] with open(manifest_file, 'r') as f: - docs = yaml.load_all(f) + docs = yaml.safe_load_all(f) for doc in docs: try: if "armada/Chart/" in doc['schema']: @@ -1363,7 +1404,7 @@ class DockerHelper(object): cutils.format_address_name(constants.CONTROLLER_HOSTNAME, constants.NETWORK_TYPE_MGMT) ).address - registry_server = '{}:{}'.format(registry_ip, common.REGISTRY_PORT) + registry_server = '{}:{}'.format(registry_ip, DOCKER_REGISTRY_PORT) return registry_server def _get_img_tag_with_registry(self, pub_img_tag): diff --git a/sysinv/sysinv/sysinv/sysinv/helm/aodh.py b/sysinv/sysinv/sysinv/sysinv/helm/aodh.py index 27233a924d..aa7edd285b 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/aodh.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/aodh.py @@ -25,7 +25,6 @@ class AodhHelm(openstack.OpenstackBaseHelm): overrides = { common.HELM_NS_OPENSTACK: { 'pod': self._get_pod_overrides(), - 'images': self._get_images_overrides(), 'conf': self._get_conf_overrides(), 'endpoints': self._get_endpoints_overrides() } @@ -50,26 +49,6 @@ class AodhHelm(openstack.OpenstackBaseHelm): } return overrides - def _get_images_overrides(self): - heat_image = self._operator.chart_operators[ - constants.HELM_CHART_HEAT].docker_image - return { - 'tags': { - 'bootstrap': heat_image, - 'db_drop': heat_image, - 'db_init': self.docker_image, - 'aodh_api': self.docker_image, - 'aodh_alarms_cleaner': self.docker_image, - 'aodh_db_sync': self.docker_image, - 'aodh_evaluator': self.docker_image, - 'aodh_listener': self.docker_image, - 'aodh_notifier': self.docker_image, - 'ks_endpoints': heat_image, - 'ks_service': heat_image, - 'ks_user': heat_image, - } - } - def _get_conf_overrides(self): return { 'aodh': { diff --git a/sysinv/sysinv/sysinv/sysinv/helm/barbican.py b/sysinv/sysinv/sysinv/sysinv/helm/barbican.py index ca020c9f0e..0652d9d331 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/barbican.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/barbican.py @@ -26,7 +26,6 @@ class BarbicanHelm(openstack.OpenstackBaseHelm): } }, 'endpoints': self._get_endpoints_overrides(), - 'images': self._get_images_overrides() } } @@ -61,20 +60,3 @@ class BarbicanHelm(openstack.OpenstackBaseHelm): self.SERVICE_NAME, self.AUTH_USERS) }, } - - def _get_images_overrides(self): - heat_image = self._operator.chart_operators[ - constants.HELM_CHART_HEAT].docker_image - return { - 'tags': { - 'barbican_api': self.docker_image, - 'barbican_db_sync': self.docker_image, - 'bootstrap': heat_image, - 'db_drop': heat_image, - 'db_init': heat_image, - 'ks_endpoints': heat_image, - 'ks_service': heat_image, - 'ks_user': heat_image, - 'scripted_test': heat_image, - } - } diff --git a/sysinv/sysinv/sysinv/sysinv/helm/base.py b/sysinv/sysinv/sysinv/sysinv/helm/base.py index bf1c5a765f..e005eddef0 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/base.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/base.py @@ -40,27 +40,6 @@ class BaseHelm(object): def context(self): return self._operator.context - @property - def docker_repo_source(self): - return common.DOCKER_SRC_LOC - - @property - def docker_repo_tag(self): - return common.DOCKER_SRCS[self.docker_repo_source][common.IMG_TAG_KEY] - - @property - def docker_image(self): - if self.docker_repo_source == common.DOCKER_SRC_LOC: - return "{}:{}/{}/{}{}:{}".format( - self._get_management_address(), common.REGISTRY_PORT, common.REPO_LOC, - common.DOCKER_SRCS[self.docker_repo_source][common.IMG_PREFIX_KEY], - self.SERVICE_NAME, self.docker_repo_tag) - else: - return "{}/{}{}:{}".format( - common.DOCKER_SRCS[self.docker_repo_source][common.IMG_BASE_KEY], - common.DOCKER_SRCS[self.docker_repo_source][common.IMG_PREFIX_KEY], - self.SERVICE_NAME, self.docker_repo_tag) - @staticmethod def quoted_str(value): return quoted_str(value) diff --git a/sysinv/sysinv/sysinv/sysinv/helm/ceilometer.py b/sysinv/sysinv/sysinv/sysinv/helm/ceilometer.py index 69065c25ec..9c53cccd73 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/ceilometer.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/ceilometer.py @@ -25,7 +25,6 @@ class CeilometerHelm(openstack.OpenstackBaseHelm): def get_overrides(self, namespace=None): overrides = { common.HELM_NS_OPENSTACK: { - 'images': self._get_images_overrides(), 'pod': self._get_pod_overrides(), 'conf': self._get_conf_overrides(), 'manifests': self._get_manifests_overrides(), @@ -41,27 +40,6 @@ class CeilometerHelm(openstack.OpenstackBaseHelm): else: return overrides - def _get_images_overrides(self): - heat_image = self._operator.chart_operators[ - constants.HELM_CHART_HEAT].docker_image - return { - 'tags': { - 'ks_service': heat_image, - 'ks_user': heat_image, - 'ks_endpoints': heat_image, - 'db_init': self.docker_image, - 'db_init_mongodb': self.docker_image, - 'ceilometer_db_sync': self.docker_image, - 'ceilometer_central': self.docker_image, - 'ceilometer_compute': self.docker_image, - 'ceilometer_ipmi': self.docker_image, - 'ceilometer_notification': self.docker_image, - 'ceilometer_collector': self.docker_image, - 'ceilometer_api': self.docker_image, - 'test': self.docker_image - } - } - def _get_pod_overrides(self): return { 'replicas': { diff --git a/sysinv/sysinv/sysinv/sysinv/helm/ceph_pools_audit.py b/sysinv/sysinv/sysinv/sysinv/helm/ceph_pools_audit.py index f402ce40ad..55771422c5 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/ceph_pools_audit.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/ceph_pools_audit.py @@ -24,14 +24,6 @@ class CephPoolsAuditHelm(base.BaseHelm): SERVICE_NAME = 'ceph-pools' - @property - def docker_repo_source(self): - return common.DOCKER_SRC_STX - - @property - def docker_repo_tag(self): - return common.DOCKER_SRCS[self.docker_repo_source][common.IMG_TAG_KEY] - def get_namespaces(self): return self.SUPPORTED_NAMESPACES diff --git a/sysinv/sysinv/sysinv/sysinv/helm/cinder.py b/sysinv/sysinv/sysinv/sysinv/helm/cinder.py index 102f7bae64..ac6b74d00d 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/cinder.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/cinder.py @@ -35,7 +35,6 @@ class CinderHelm(openstack.OpenstackBaseHelm): 'backup': self._num_controllers() } }, - 'images': self._get_images_overrides(), 'conf': { 'cinder': self._get_conf_cinder_overrides(), 'ceph': self._get_conf_ceph_overrides(), @@ -181,36 +180,6 @@ class CinderHelm(openstack.OpenstackBaseHelm): } - def _get_images_overrides(self): - heat_image = self._operator.chart_operators[ - constants.HELM_CHART_HEAT].docker_image - - # TODO: Remove after ceph upgrade - # Format the name of the stx specific ceph config helper - ceph_config_helper_image = "{}:{}/{}/{}{}:{}".format( - self._get_management_address(), common.REGISTRY_PORT, common.REPO_LOC, - common.DOCKER_SRCS[self.docker_repo_source][common.IMG_PREFIX_KEY], - 'ceph-config-helper', self.docker_repo_tag) - - return { - 'tags': { - 'bootstrap': heat_image, - 'cinder_api': self.docker_image, - 'cinder_backup': self.docker_image, - 'cinder_backup_storage_init': ceph_config_helper_image, - 'cinder_db_sync': self.docker_image, - 'cinder_scheduler': self.docker_image, - 'cinder_storage_init': ceph_config_helper_image, - 'cinder_volume': self.docker_image, - 'cinder_volume_usage_audit': self.docker_image, - 'db_drop': heat_image, - 'db_init': heat_image, - 'ks_endpoints': heat_image, - 'ks_service': heat_image, - 'ks_user': heat_image, - } - } - def _get_primary_ceph_backend(self): try: backend = self.dbapi.storage_backend_get_by_name( diff --git a/sysinv/sysinv/sysinv/sysinv/helm/common.py b/sysinv/sysinv/sysinv/sysinv/helm/common.py index 587d808e5f..1bfec2818d 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/common.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/common.py @@ -47,51 +47,3 @@ LABEL_REMOTE_STORAGE = 'remote-storage' # Label values LABEL_VALUE_ENABLED = 'enabled' LABEL_VALUE_DISABLED = 'disabled' - -# Image Tagging -DOCKER_SRC_OSH = 'openstackhelm' -DOCKER_SRC_STX = 'starlingx' -DOCKER_SRC_LOC = 'controller' - -# TODO (rchurch): These values and settings are currently provided for early -# integration scenarios. As we formalize delivery mechanisms, these will need to -# be adjusted accordingly. -REGISTRY_PORT = '9001' -REGISTRY_OSH = 'docker.io' -REGISTRY_STX = '128.224.186.231:%s' % REGISTRY_PORT -REGISTRY_LOC = '192.168.204.2:%s' % REGISTRY_PORT - -REPO_OSH = 'openstackhelm' -REPO_STX = 'abailey' -REPO_LOC = 'starlingx' - -IMG_PREFIX_KEY = 'prefix' -IMG_BASE_KEY = 'base' -IMG_TAG_KEY = 'tag' - -IMG_PREFIX_OSH = '' -IMG_PREFIX_STX = 'stx-' -IMG_PREFIX_LOC = 'stx-' - -TAGS_PIKE = 'pike' -TAGS_LATEST = 'latest' -TAGS_STX_LATEST = 'master-centos-dev-latest' - -DOCKER_SRCS = { - DOCKER_SRC_OSH: { - IMG_BASE_KEY: '{}/{}'.format(REGISTRY_OSH, REPO_OSH), - IMG_PREFIX_KEY: IMG_PREFIX_OSH, - IMG_TAG_KEY: TAGS_PIKE - - }, - DOCKER_SRC_STX: { - IMG_BASE_KEY: '{}/{}'.format(REGISTRY_STX, REPO_STX), - IMG_PREFIX_KEY: IMG_PREFIX_STX, - IMG_TAG_KEY: TAGS_STX_LATEST - }, - DOCKER_SRC_LOC: { - IMG_BASE_KEY: '{}/{}'.format(REGISTRY_LOC, REPO_LOC), - IMG_PREFIX_KEY: IMG_PREFIX_LOC, - IMG_TAG_KEY: TAGS_STX_LATEST - } -} diff --git a/sysinv/sysinv/sysinv/sysinv/helm/garbd.py b/sysinv/sysinv/sysinv/sysinv/helm/garbd.py index fdb4501c6a..3b500674fa 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/garbd.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/garbd.py @@ -66,7 +66,6 @@ class GarbdHelm(base.BaseHelm): def get_overrides(self, namespace=None): overrides = { common.HELM_NS_OPENSTACK: { - 'images': self._get_images_overrides(), } } @@ -77,11 +76,3 @@ class GarbdHelm(base.BaseHelm): namespace=namespace) else: return overrides - - def _get_images_overrides(self): - - return { - 'tags': { - 'garbd': self.docker_image - } - } diff --git a/sysinv/sysinv/sysinv/sysinv/helm/glance.py b/sysinv/sysinv/sysinv/sysinv/helm/glance.py index a414e86b4b..89b528383b 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/glance.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/glance.py @@ -35,7 +35,6 @@ class GlanceHelm(openstack.OpenstackBaseHelm): 'endpoints': self._get_endpoints_overrides(), 'storage': self._get_storage_overrides(), 'conf': self._get_conf_overrides(), - 'images': self._get_images_overrides(), 'bootstrap': self._get_bootstrap_overrides() } } @@ -48,30 +47,6 @@ class GlanceHelm(openstack.OpenstackBaseHelm): else: return overrides - def _get_images_overrides(self): - heat_image = self._operator.chart_operators[ - constants.HELM_CHART_HEAT].docker_image - ceph_config_helper_image = "{}:{}/{}/{}{}:{}".format( - self._get_management_address(), common.REGISTRY_PORT, - common.REPO_LOC, - common.DOCKER_SRCS[self.docker_repo_source][common.IMG_PREFIX_KEY], - 'ceph-config-helper', self.docker_repo_tag) - - return { - 'tags': { - 'bootstrap': heat_image, - 'db_drop': heat_image, - 'db_init': heat_image, - 'glance_storage_init': ceph_config_helper_image, - 'glance_api': self.docker_image, - 'glance_db_sync': self.docker_image, - 'glance_registry': self.docker_image, - 'ks_endpoints': heat_image, - 'ks_service': heat_image, - 'ks_user': heat_image, - } - } - def _get_pod_overrides(self): replicas_count = 1 ceph_backend = self._get_primary_ceph_backend() diff --git a/sysinv/sysinv/sysinv/sysinv/helm/gnocchi.py b/sysinv/sysinv/sysinv/sysinv/helm/gnocchi.py index 56246a6929..d69141c34f 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/gnocchi.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/gnocchi.py @@ -24,7 +24,6 @@ class GnocchiHelm(openstack.OpenstackBaseHelm): def get_overrides(self, namespace=None): overrides = { common.HELM_NS_OPENSTACK: { - 'images': self._get_images_overrides(), 'pod': self._get_pod_overrides(), 'endpoints': self._get_endpoints_overrides(), } @@ -38,24 +37,6 @@ class GnocchiHelm(openstack.OpenstackBaseHelm): else: return overrides - def _get_images_overrides(self): - heat_image = self._operator.chart_operators[ - constants.HELM_CHART_HEAT].docker_image - return { - 'tags': { - 'db_init': self.docker_image, - 'db_init_indexer': self.docker_image, - 'db_sync': self.docker_image, - 'gnocchi_api': self.docker_image, - 'gnocchi_metricd': self.docker_image, - 'gnocchi_resources_cleaner': self.docker_image, - 'gnocchi_statsd': self.docker_image, - 'ks_endpoints': heat_image, - 'ks_service': heat_image, - 'ks_user': heat_image, - } - } - def _get_pod_overrides(self): return { 'replicas': { diff --git a/sysinv/sysinv/sysinv/sysinv/helm/heat.py b/sysinv/sysinv/sysinv/sysinv/helm/heat.py index d988b88b60..5a54bc405c 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/heat.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/heat.py @@ -26,7 +26,6 @@ class HeatHelm(openstack.OpenstackBaseHelm): common.HELM_NS_OPENSTACK: { 'pod': self._get_pod_overrides(), 'endpoints': self._get_endpoints_overrides(), - 'images': self._get_images_overrides(), } } @@ -38,25 +37,6 @@ class HeatHelm(openstack.OpenstackBaseHelm): else: return overrides - def _get_images_overrides(self): - return { - 'tags': { - 'bootstrap': self.docker_image, - 'db_drop': self.docker_image, - 'db_init': self.docker_image, - 'heat_api': self.docker_image, - 'heat_cfn': self.docker_image, - 'heat_cloudwatch': self.docker_image, - 'heat_db_sync': self.docker_image, - 'heat_engine': self.docker_image, - 'heat_engine_cleaner': self.docker_image, - 'heat_purge_deleted': self.docker_image, - 'ks_endpoints': self.docker_image, - 'ks_service': self.docker_image, - 'ks_user': self.docker_image, - } - } - def _get_pod_overrides(self): return { 'replicas': { diff --git a/sysinv/sysinv/sysinv/sysinv/helm/helm.py b/sysinv/sysinv/sysinv/sysinv/helm/helm.py index d8793e8da0..7d85577fd4 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/helm.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/helm.py @@ -44,26 +44,12 @@ def helm_context(func): class HelmOperator(object): """Class to encapsulate helm override operations for System Inventory""" - def __init__(self, dbapi=None, path=None, docker_repository=None): + def __init__(self, dbapi=None, path=None): if path is None: path = common.HELM_OVERRIDES_PATH - # Set the primary source of docker images - if docker_repository is None: - # During initial development, use upstream OSH images by default and - # switch to the STX repo when the images are validated and ready for - # use. - docker_repository = common.DOCKER_SRC_OSH - else: - valid_docker_repositories = common.DOCKER_SRCS.keys() - if docker_repository not in valid_docker_repositories: - raise exception.InvalidHelmDockerImageSource( - source=docker_repository, - valid_srcs=valid_docker_repositories) - self.dbapi = dbapi self.path = path - self.docker_repo_source = docker_repository # register chart operators for lookup self.chart_operators = {} diff --git a/sysinv/sysinv/sysinv/sysinv/helm/horizon.py b/sysinv/sysinv/sysinv/sysinv/helm/horizon.py index 80837896f0..806eeada53 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/horizon.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/horizon.py @@ -23,7 +23,6 @@ class HorizonHelm(openstack.OpenstackBaseHelm): def get_overrides(self, namespace=None): overrides = { common.HELM_NS_OPENSTACK: { - 'images': self._get_images_overrides(), 'conf': { 'horizon': { 'local_settings': { @@ -63,18 +62,6 @@ class HorizonHelm(openstack.OpenstackBaseHelm): }, } - def _get_images_overrides(self): - heat_image = self._operator.chart_operators[ - constants.HELM_CHART_HEAT].docker_image - return { - 'tags': { - 'db_drop': heat_image, - 'db_init': heat_image, - 'horizon': self.docker_image, - 'horizon_db_sync': self.docker_image, - } - } - def _get_local_settings_config_overrides(self): local_settings_config = { 'horizon_secret_key': self._get_or_generate_password( diff --git a/sysinv/sysinv/sysinv/sysinv/helm/ironic.py b/sysinv/sysinv/sysinv/sysinv/helm/ironic.py index c2142e78ef..6796df46dc 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/ironic.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/ironic.py @@ -21,7 +21,6 @@ class IronicHelm(openstack.OpenstackBaseHelm): def get_overrides(self, namespace=None): overrides = { common.HELM_NS_OPENSTACK: { - 'images': self._get_images_overrides(), } } @@ -32,24 +31,3 @@ class IronicHelm(openstack.OpenstackBaseHelm): namespace=namespace) else: return overrides - - def _get_images_overrides(self): - heat_image = self._operator.chart_operators[ - constants.HELM_CHART_HEAT].docker_image - return { - 'tags': { - 'bootstrap': heat_image, - 'db_init': heat_image, - 'ironic_api': self.docker_image, - 'ironic_conductor': self.docker_image, - 'ironic_db_sync': self.docker_image, - 'ironic_manage_cleaning_network': heat_image, - 'ironic_pxe': self.docker_image, - 'ironic_pxe_init': self.docker_image, - 'ironic_retrive_cleaning_network': heat_image, - 'ironic_retrive_swift_config': heat_image, - 'ks_endpoints': heat_image, - 'ks_service': heat_image, - 'ks_user': heat_image, - } - } diff --git a/sysinv/sysinv/sysinv/sysinv/helm/keystone.py b/sysinv/sysinv/sysinv/sysinv/helm/keystone.py index 1fb3db0c8c..9bc1e2250b 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/keystone.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/keystone.py @@ -35,7 +35,6 @@ class KeystoneHelm(openstack.OpenstackBaseHelm): 'pod': self._get_pod_overrides(), 'conf': self._get_conf_overrides(), 'endpoints': self._get_endpoints_overrides(), - 'images': self._get_images_overrides(), } } @@ -60,29 +59,8 @@ class KeystoneHelm(openstack.OpenstackBaseHelm): } } } - if self.docker_repo_source != common.DOCKER_SRC_OSH: - overrides.update({'user': {'keystone': {'uid': 0}}}) return overrides - def _get_images_overrides(self): - heat_image = self._operator.chart_operators[ - constants.HELM_CHART_HEAT].docker_image - return { - 'tags': { - 'bootstrap': heat_image, - 'db_drop': heat_image, - 'db_init': heat_image, - 'keystone_api': self.docker_image, - 'keystone_credential_rotate': self.docker_image, - 'keystone_credential_setup': self.docker_image, - 'keystone_db_sync': self.docker_image, - 'keystone_domain_manage': self.docker_image, - 'keystone_fernet_rotate': self.docker_image, - 'keystone_fernet_setup': self.docker_image, - 'ks_user': heat_image, - } - } - def _get_conf_keystone_default_overrides(self): return { 'max_token_size': 255, # static controller.yaml => chart default diff --git a/sysinv/sysinv/sysinv/sysinv/helm/libvirt.py b/sysinv/sysinv/sysinv/sysinv/helm/libvirt.py index 5631d0f45c..a6a48553cb 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/libvirt.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/libvirt.py @@ -41,8 +41,6 @@ class LibvirtHelm(openstack.OpenstackBaseHelm): } } - self._get_images_overrides(overrides[common.HELM_NS_OPENSTACK]) - if namespace in self.SUPPORTED_NAMESPACES: return overrides[namespace] elif namespace: @@ -50,13 +48,3 @@ class LibvirtHelm(openstack.OpenstackBaseHelm): namespace=namespace) else: return overrides - - def _get_images_overrides(self, overrides_dict): - if self.docker_repo_source != common.DOCKER_SRC_OSH: - overrides_dict.update({ - 'images': { - 'tags': { - 'libvirt': self.docker_image - } - } - }) diff --git a/sysinv/sysinv/sysinv/sysinv/helm/magnum.py b/sysinv/sysinv/sysinv/sysinv/helm/magnum.py index 773c0338dc..6a51a0df69 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/magnum.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/magnum.py @@ -21,7 +21,6 @@ class MagnumHelm(openstack.OpenstackBaseHelm): def get_overrides(self, namespace=None): overrides = { common.HELM_NS_OPENSTACK: { - 'images': self._get_images_overrides(), 'pod': { 'replicas': { 'api': self._num_controllers(), @@ -38,20 +37,3 @@ class MagnumHelm(openstack.OpenstackBaseHelm): namespace=namespace) else: return overrides - - def _get_images_overrides(self): - heat_image = self._operator.chart_operators[ - constants.HELM_CHART_HEAT].docker_image - return { - 'tags': { - 'bootstrap': heat_image, - 'db_drop': heat_image, - 'db_init': heat_image, - 'ks_endpoints': heat_image, - 'ks_service': heat_image, - 'ks_user': heat_image, - 'magnum_api': self.docker_image, - 'magnum_conductor': self.docker_image, - 'magnum_db_sync': self.docker_image, - } - } diff --git a/sysinv/sysinv/sysinv/sysinv/helm/mariadb.py b/sysinv/sysinv/sysinv/sysinv/helm/mariadb.py index 6d56ca2c5f..572f2c2793 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/mariadb.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/mariadb.py @@ -36,7 +36,6 @@ class MariadbHelm(openstack.OpenstackBaseHelm): 'ingress': self._num_controllers() } }, - 'images': self._get_images_overrides(), 'endpoints': self._get_endpoints_overrides(), } } @@ -49,16 +48,6 @@ class MariadbHelm(openstack.OpenstackBaseHelm): else: return overrides - def _get_images_overrides(self): - heat_image = self._operator.chart_operators[ - constants.HELM_CHART_HEAT].docker_image - - return { - 'tags': { - 'prometheus_mysql_exporter_helm_tests': heat_image - } - } - def _get_endpoints_overrides(self): return { 'oslo_db': { diff --git a/sysinv/sysinv/sysinv/sysinv/helm/neutron.py b/sysinv/sysinv/sysinv/sysinv/helm/neutron.py index 9bf672e990..4ef09fc246 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/neutron.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/neutron.py @@ -65,7 +65,6 @@ class NeutronHelm(openstack.OpenstackBaseHelm): }, }, 'endpoints': self._get_endpoints_overrides(), - 'images': self._get_images_overrides(), } } @@ -240,7 +239,7 @@ class NeutronHelm(openstack.OpenstackBaseHelm): # obtain the assigned bridge for interface brname = 'br-phy%d' % index port_name = self._get_interface_port_name(iface) - bridges[brname] = port_name + bridges[brname] = port_name.encode('utf8', 'strict') index += 1 return bridges @@ -384,32 +383,6 @@ class NeutronHelm(openstack.OpenstackBaseHelm): return None - def _get_images_overrides(self): - heat_image = self._operator.chart_operators[ - constants.HELM_CHART_HEAT].docker_image - return { - 'tags': { - 'bootstrap': heat_image, - 'db_init': heat_image, - 'neutron_db_sync': self.docker_image, - 'db_drop': heat_image, - 'ks_user': heat_image, - 'ks_service': heat_image, - 'ks_endpoints': heat_image, - 'neutron_server': self.docker_image, - 'neutron_dhcp': self.docker_image, - 'neutron_metadata': self.docker_image, - 'neutron_l3': self.docker_image, - 'neutron_openvswitch_agent': self.docker_image, - 'neutron_linuxbridge_agent': self.docker_image, - # TODO (rchurch): Fix this... Suffix tied to a release??? - # 'neutron_sriov_agent': '{}{}'.format(self.docker_image,'-sriov-1804'), - # 'neutron_sriov_agent_init': '{}{}'.format(self.docker_image,'-sriov-1804'), - 'neutron_sriov_agent': self.docker_image, - 'neutron_sriov_agent_init': self.docker_image, - } - } - def _get_endpoints_overrides(self): overrides = { 'identity': { diff --git a/sysinv/sysinv/sysinv/sysinv/helm/nova.py b/sysinv/sysinv/sysinv/sysinv/helm/nova.py index 513005af6d..4a6f5909ed 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/nova.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/nova.py @@ -97,7 +97,6 @@ class NovaHelm(openstack.OpenstackBaseHelm): 'ssh_public': ssh_publickey, }, 'endpoints': self._get_endpoints_overrides(), - 'images': self._get_images_overrides(), 'network': { 'sshd': { 'from_subnet': self._get_ssh_subnet(), @@ -114,44 +113,6 @@ class NovaHelm(openstack.OpenstackBaseHelm): else: return overrides - def _get_images_overrides(self): - heat_image = self._operator.chart_operators[ - constants.HELM_CHART_HEAT].docker_image - - # TODO: Remove after ceph upgrade - # Format the name of the stx specific ceph config helper - ceph_config_helper_image = "{}:{}/{}/{}{}:{}".format( - self._get_management_address(), common.REGISTRY_PORT, - common.REPO_LOC, - common.DOCKER_SRCS[self.docker_repo_source][common.IMG_PREFIX_KEY], - 'ceph-config-helper', self.docker_repo_tag) - - return { - 'tags': { - 'bootstrap': heat_image, - 'db_drop': heat_image, - 'db_init': heat_image, - 'ks_user': heat_image, - 'ks_service': heat_image, - 'ks_endpoints': heat_image, - 'nova_api': self.docker_image, - 'nova_cell_setup': self.docker_image, - 'nova_cell_setup_init': heat_image, - 'nova_compute': self.docker_image, - 'nova_compute_ironic': self.docker_image, - 'nova_compute_ssh': self.docker_image, - 'nova_conductor': self.docker_image, - 'nova_consoleauth': self.docker_image, - 'nova_db_sync': self.docker_image, - 'nova_novncproxy': self.docker_image, - 'nova_placement': self.docker_image, - 'nova_scheduler': self.docker_image, - 'nova_spiceproxy': self.docker_image, - 'nova_spiceproxy_assets': self.docker_image, - 'nova_storage_init': ceph_config_helper_image, - } - } - def _get_endpoints_overrides(self): overrides = { 'identity': { diff --git a/sysinv/sysinv/sysinv/sysinv/helm/nova_api_proxy.py b/sysinv/sysinv/sysinv/sysinv/helm/nova_api_proxy.py index 1cf18093d8..7f65294a6f 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/nova_api_proxy.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/nova_api_proxy.py @@ -39,7 +39,6 @@ class NovaApiProxyHelm(openstack.OpenstackBaseHelm): }, } }, - 'images': self._get_images_overrides(), 'endpoints': self._get_endpoints_overrides(), } } @@ -52,17 +51,6 @@ class NovaApiProxyHelm(openstack.OpenstackBaseHelm): else: return overrides - def _get_images_overrides(self): - heat_image = self._operator.chart_operators[ - constants.HELM_CHART_HEAT].docker_image - - return { - 'tags': { - 'nova_api_proxy': self.docker_image, - 'ks_endpoints': heat_image - } - } - def _get_endpoints_overrides(self): nova_service_name = self._operator.chart_operators[ constants.HELM_CHART_NOVA].SERVICE_NAME diff --git a/sysinv/sysinv/sysinv/sysinv/helm/panko.py b/sysinv/sysinv/sysinv/sysinv/helm/panko.py index 9a73132478..351004cf7e 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/panko.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/panko.py @@ -25,7 +25,6 @@ class PankoHelm(openstack.OpenstackBaseHelm): overrides = { common.HELM_NS_OPENSTACK: { 'pod': self._get_pod_overrides(), - 'images': self._get_images_overrides(), 'endpoints': self._get_endpoints_overrides() } } @@ -46,23 +45,6 @@ class PankoHelm(openstack.OpenstackBaseHelm): } return overrides - def _get_images_overrides(self): - heat_image = self._operator.chart_operators[ - constants.HELM_CHART_HEAT].docker_image - return { - 'tags': { - 'bootstrap': heat_image, - 'db_drop': heat_image, - 'db_init': self.docker_image, - 'panko_db_sync': self.docker_image, - 'panko_api': self.docker_image, - 'panko_events_cleaner': self.docker_image, - 'ks_endpoints': heat_image, - 'ks_service': heat_image, - 'ks_user': heat_image, - } - } - def _get_endpoints_overrides(self): return { 'identity': { diff --git a/sysinv/sysinv/sysinv/sysinv/helm/rabbitmq.py b/sysinv/sysinv/sysinv/sysinv/helm/rabbitmq.py index a6455cec4f..4b6281a54a 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/rabbitmq.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/rabbitmq.py @@ -26,7 +26,6 @@ class RabbitmqHelm(openstack.OpenstackBaseHelm): 'server': self._num_controllers() } }, - 'images': self._get_images_overrides(), 'endpoints': self._get_endpoints_overrides(), } } @@ -39,16 +38,6 @@ class RabbitmqHelm(openstack.OpenstackBaseHelm): else: return overrides - def _get_images_overrides(self): - heat_image = self._operator.chart_operators[ - constants.HELM_CHART_HEAT].docker_image - - return { - 'tags': { - 'prometheus_rabbitmq_exporter_helm_tests': heat_image - } - } - def _get_endpoints_overrides(self): credentials = self._get_endpoints_oslo_messaging_overrides( self.CHART, []) diff --git a/sysinv/sysinv/sysinv/sysinv/helm/rbd_provisioner.py b/sysinv/sysinv/sysinv/sysinv/helm/rbd_provisioner.py index 55b46a2ef9..d7232840b6 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/rbd_provisioner.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/rbd_provisioner.py @@ -73,7 +73,6 @@ class RbdProvisionerHelm(base.BaseHelm): common.HELM_NS_OPENSTACK: { "classdefaults": classdefaults, "classes": classes, - "images": self._get_images_overrides(), "global": global_settings } } @@ -85,18 +84,3 @@ class RbdProvisionerHelm(base.BaseHelm): namespace=namespace) else: return overrides - - def _get_images_overrides(self): - # TODO: Remove after ceph upgrade - # Format the name of the stx specific ceph config helper - local_docker_registry_ip = self._get_management_address() - ceph_config_helper_image = "{}:{}/{}/{}{}:{}".format( - local_docker_registry_ip, common.REGISTRY_PORT, common.REPO_LOC, - common.DOCKER_SRCS[self.docker_repo_source][common.IMG_PREFIX_KEY], - 'ceph-config-helper', self.docker_repo_tag) - - return { - 'tags': { - 'rbd_provisioner_storage_init': ceph_config_helper_image, - } - }