From 6ad6bb6d8c9b162540b689c9e8b9385e847c922a Mon Sep 17 00:00:00 2001 From: "Crank, Daniel (dc6350)" Date: Fri, 16 Apr 2021 14:35:37 -0500 Subject: [PATCH] Move vino-builder into vino This patchset moves the vino-builder image build from the images repo into the vino repo. [0] removes it from the images repo. [0] https://review.opendev.org/c/airship/images/+/786714 Change-Id: I8299045cdfaaf285e0f088b330a84e00fcb8307c --- Makefile | 10 + config/manager/daemonset-template.yaml | 2 +- playbooks/integration-test-airshipctl.yaml | 1 + roles/vino-build-images/tasks/main.yaml | 14 ++ roles/vino-publish-images/tasks/main.yaml | 7 +- tools/deployment/deploy-vino.sh | 1 + vino-builder/Dockerfile | 45 ++++ vino-builder/assets/entrypoint.sh | 70 +++++++ .../playbooks/roles/libvirt/defaults/main.yml | 1 + .../roles/libvirt/library/core_allocation.py | 160 ++++++++++++++ .../roles/libvirt/tasks/create-domain.yaml | 52 +++++ .../roles/libvirt/tasks/create-network.yaml | 36 ++++ .../roles/libvirt/tasks/create-storage.yaml | 18 ++ .../playbooks/roles/libvirt/tasks/main.yml | 39 ++++ .../playbooks/sample-vino-ansible-input.yaml | 198 ++++++++++++++++++ .../assets/playbooks/vino-builder.yaml | 46 ++++ 16 files changed, 698 insertions(+), 2 deletions(-) create mode 100644 vino-builder/Dockerfile create mode 100644 vino-builder/assets/entrypoint.sh create mode 100644 vino-builder/assets/playbooks/roles/libvirt/defaults/main.yml create mode 100644 vino-builder/assets/playbooks/roles/libvirt/library/core_allocation.py create mode 100644 vino-builder/assets/playbooks/roles/libvirt/tasks/create-domain.yaml create mode 100644 vino-builder/assets/playbooks/roles/libvirt/tasks/create-network.yaml create mode 100644 vino-builder/assets/playbooks/roles/libvirt/tasks/create-storage.yaml create mode 100644 vino-builder/assets/playbooks/roles/libvirt/tasks/main.yml create mode 100644 vino-builder/assets/playbooks/sample-vino-ansible-input.yaml create mode 100644 vino-builder/assets/playbooks/vino-builder.yaml diff --git a/Makefile b/Makefile index 79a685f..f402165 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ # IMG ?= controller:latest CONTROLLER_IMG ?= quay.io/airshipit/vino NODE_LABELER_IMG ?= quay.io/airshipit/nodelabeler +VINO_BUILDER_IMG ?= quay.io/airshipit/vino-builder # Produce CRDs that work back to Kubernetes 1.16 CRD_OPTIONS ?= crd:crdVersions=v1 @@ -78,6 +79,11 @@ docker-build-controller: docker-build-nodelabeler: docker build -f nodelabeler/Dockerfile . ${DOCKER_PROXY_FLAGS} -t ${NODE_LABELER_IMG} +# Build the vino-builder docker image +# If DOCKER_PROXY_FLAGS values are empty, we are fine with that +docker-build-vino-builder: + docker build -f vino-builder/Dockerfile . ${DOCKER_PROXY_FLAGS} -t ${VINO_BUILDER_IMG} + # Push the controller docker image docker-push-controller: docker push ${CONTROLLER_IMG} @@ -86,6 +92,10 @@ docker-push-controller: docker-push-nodelabeler: docker push ${NODE_LABELER_IMG} +# Push the vino-builder docker image +docker-push-vino-builder: + docker push ${VINO_BUILDER_IMG} + # Generate API reference documentation api-docs: gen-crd-api-reference-docs $(API_REF_GEN) -api-dir=./pkg/api/v1 -config=./hack/api-docs/config.json -template-dir=./hack/api-docs/template -out-file=./docs/api/vino.md diff --git a/config/manager/daemonset-template.yaml b/config/manager/daemonset-template.yaml index f6a3c76..86432db 100644 --- a/config/manager/daemonset-template.yaml +++ b/config/manager/daemonset-template.yaml @@ -114,7 +114,7 @@ spec: ports: - containerPort: 8001 hostPort: 8001 - image: quay.io/airshipit/vino-builder:latest-ubuntu_bionic + image: quay.io/airshipit/vino-builder imagePullPolicy: IfNotPresent volumeMounts: - name: flavors diff --git a/playbooks/integration-test-airshipctl.yaml b/playbooks/integration-test-airshipctl.yaml index 1cce7da..199d117 100644 --- a/playbooks/integration-test-airshipctl.yaml +++ b/playbooks/integration-test-airshipctl.yaml @@ -22,6 +22,7 @@ ./tools/deployment/install-airship.sh ./tools/deployment/configure-airship.sh make docker-build-controller + make docker-build-vino-builder ./tools/deployment/run-test-plan.sh args: chdir: "{{ zuul.project.src_dir }}" diff --git a/roles/vino-build-images/tasks/main.yaml b/roles/vino-build-images/tasks/main.yaml index 597a602..f4c0d85 100644 --- a/roles/vino-build-images/tasks/main.yaml +++ b/roles/vino-build-images/tasks/main.yaml @@ -39,5 +39,19 @@ - name: Verify nodelabeler image exists shell: docker image inspect quay.io/airshipit/nodelabeler + args: + chdir: "{{ zuul.project.src_dir }}" + +- name: Buid vino-builder image + make: + chdir: "{{ zuul.project.src_dir }}" + target: docker-build-vino-builder + params: + PROXY: "{{ proxy.http }}" + NO_PROXY: "{{ proxy.noproxy }}" + USE_PROXY: "{{ proxy.enabled | lower }}" + +- name: Verify vino-builder image exists + shell: docker image inspect quay.io/airshipit/vino-builder args: chdir: "{{ zuul.project.src_dir }}" \ No newline at end of file diff --git a/roles/vino-publish-images/tasks/main.yaml b/roles/vino-publish-images/tasks/main.yaml index 2bd8e36..8e6f3ac 100644 --- a/roles/vino-publish-images/tasks/main.yaml +++ b/roles/vino-publish-images/tasks/main.yaml @@ -44,4 +44,9 @@ - name: Push nodelabeler image with latest tag make: chdir: "{{ zuul.project.src_dir }}" - target: docker-push-nodelabeler \ No newline at end of file + target: docker-push-nodelabeler + + - name: Push vino-builder image with latest tag + make: + chdir: "{{ zuul.project.src_dir }}" + target: docker-push-vino-builder \ No newline at end of file diff --git a/tools/deployment/deploy-vino.sh b/tools/deployment/deploy-vino.sh index 7dd4f98..e8130c9 100755 --- a/tools/deployment/deploy-vino.sh +++ b/tools/deployment/deploy-vino.sh @@ -3,6 +3,7 @@ set -xe sudo snap install kustomize && sudo snap install go --classic make docker-build-controller +make docker-build-vino-builder make deploy kubectl get po -A #Wait for vino controller manager Pod. diff --git a/vino-builder/Dockerfile b/vino-builder/Dockerfile new file mode 100644 index 0000000..7450c60 --- /dev/null +++ b/vino-builder/Dockerfile @@ -0,0 +1,45 @@ +FROM ubuntu:18.04 + +SHELL ["bash", "-exc"] +ENV DEBIAN_FRONTEND noninteractive + +ARG k8s_version=v1.18.3 +ARG kubectl_url=https://storage.googleapis.com/kubernetes-release/release/"${k8s_version}"/bin/linux/amd64/kubectl + + +# Update distro and install common reqs +RUN apt-get update ;\ + apt-get dist-upgrade -y ;\ + apt-get install -y \ + python3-minimal \ + python3-pip \ + python3-setuptools \ + python3-libvirt \ + libvirt-clients \ + python3-netaddr \ + python3-lxml \ + curl \ + make \ + sudo \ + iproute2 \ + bridge-utils \ + iputils-ping \ + net-tools \ + less \ + jq \ + vim \ + openssh-client ;\ + curl -sSLo /usr/local/bin/kubectl "${kubectl_url}" ;\ + chmod +x /usr/local/bin/kubectl ;\ + pip3 install --upgrade pip ;\ + pip3 install --upgrade wheel ;\ + pip3 install --upgrade ansible ;\ + rm -rf /var/lib/apt/lists/* + +COPY vino-builder/assets /opt/assets/ +RUN cp -ravf /opt/assets/* / ;\ + rm -rf /opt/assets + +RUN chmod +x /entrypoint.sh + +ENTRYPOINT /entrypoint.sh diff --git a/vino-builder/assets/entrypoint.sh b/vino-builder/assets/entrypoint.sh new file mode 100644 index 0000000..b089f50 --- /dev/null +++ b/vino-builder/assets/entrypoint.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +READINESS_CHECK_FILE="/tmp/healthy" + +## Remove healthy status before starting +[ -f "${READINESS_CHECK_FILE}" ] && rm ${READINESS_CHECK_FILE} + +# wait for libvirt socket to be ready +TIMEOUT=300 +while [[ ! -e /var/run/libvirt/libvirt-sock ]]; do + if [[ ${TIMEOUT} -gt 0 ]]; then + let TIMEOUT-=1 + echo "Waiting for libvirt socket at /var/run/libvirt/libvirt-sock" + sleep 1 + else + echo "ERROR: libvirt did not start in time (socket missing) /var/run/libvirt/libvirt-sock" + exit 1 + fi +done + +# wait for dynamic data to be ready +# data is node-specific, so it will be passed as a node annotations +# of the form +# metadata: +# annotations: +# airshipit.org/vino.network-values: | +# bunch-of-yaml +DYNAMIC_DATA_FILE=/var/lib/vino-builder/dynamic.yaml +TIMEOUT=300 +while [[ ${TIMEOUT} -gt 0 ]]; do + let TIMEOUT-=10 + if [[ ${TIMEOUT} -le 0 ]]; then + echo "ERROR: vino-builder dynamic data was not ready in time" + exit 1 + fi + kubectl get node $HOSTNAME -o=jsonpath="{.metadata.annotations.airshipit\.org/vino\.network-values}" > $DYNAMIC_DATA_FILE + if [[ -s $DYNAMIC_DATA_FILE ]]; then + break + fi + echo "vino-builder dynamic data not ready yet - sleeping for 10 seconds..." + sleep 10 +done + +ansible-playbook -v \ + -e @/var/lib/vino-builder/flavors/flavors.yaml \ + -e @/var/lib/vino-builder/flavor-templates/flavor-templates.yaml \ + -e @/var/lib/vino-builder/network-templates/network-templates.yaml \ + -e @/var/lib/vino-builder/storage-templates/storage-templates.yaml \ + -e @$DYNAMIC_DATA_FILE \ + /playbooks/vino-builder.yaml + +touch ${READINESS_CHECK_FILE} + +while true; do + sleep infinity +done \ No newline at end of file diff --git a/vino-builder/assets/playbooks/roles/libvirt/defaults/main.yml b/vino-builder/assets/playbooks/roles/libvirt/defaults/main.yml new file mode 100644 index 0000000..d2a4f49 --- /dev/null +++ b/vino-builder/assets/playbooks/roles/libvirt/defaults/main.yml @@ -0,0 +1 @@ +libvirt_uri: qemu:///system \ No newline at end of file diff --git a/vino-builder/assets/playbooks/roles/libvirt/library/core_allocation.py b/vino-builder/assets/playbooks/roles/libvirt/library/core_allocation.py new file mode 100644 index 0000000..eb25684 --- /dev/null +++ b/vino-builder/assets/playbooks/roles/libvirt/library/core_allocation.py @@ -0,0 +1,160 @@ +#!/usr/bin/python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# generate_baremetal_macs method ripped from +# openstack/tripleo-incubator/scripts/configure-vm + +import math +import random +import sys +import fnmatch +import os +from itertools import chain +import json + +DOCUMENTATION = ''' +--- +module: core_allocation +version_added: "1.0" +short_description: Allocate numa aligned cores for libvirt domains and track allocations +description: + - Generate numa aligned cores for libvirt domains and track allocations +''' + +PATH_SYS_DEVICES_NODE = "/sys/devices/system/node" + +def _parse_range(rng): + parts = rng.split('-') + if 1 > len(parts) > 2: + raise ValueError("Bad range: '%s'" % (rng,)) + parts = [int(i) for i in parts] + start = parts[0] + end = start if len(parts) == 1 else parts[1] + if start > end: + end, start = start, end + return range(start, end + 1) + +def _parse_range_list(rngs): + return sorted(set(chain(*[_parse_range(rng) for rng in rngs.split(',')]))) + +def get_numa_cores(): + """Return cores as a dict of numas each with their expanded core lists""" + numa_core_dict = {} + for root, dir, files in os.walk(PATH_SYS_DEVICES_NODE): + for numa in fnmatch.filter(dir, "node*"): + numa_path = os.path.join(PATH_SYS_DEVICES_NODE, numa) + cpulist = os.path.join(numa_path, "cpulist") + with open(cpulist, 'r') as f: + parsed_range_list = _parse_range_list(f.read()) + numa_core_dict[numa] = parsed_range_list + return numa_core_dict + +def allocate_cores(nodes, flavors, exclude_cpu): + """Return""" + + core_state = {} + + try: + f = open('/etc/libvirt/vino-cores.json', 'r') + core_state = json.loads(f.read()) + except: + pass + + # instantiate initial inventory - we don't support the inventory + # changing (e.g. adding cores) + if 'inventory' not in core_state: + core_state['inventory'] = get_numa_cores() + + # explode exclude cpu list - we don't support adjusting this after-the-fact + # right now + if 'exclude' not in core_state: + exclude_core_list = _parse_range_list(exclude_cpu) + core_state['exclude'] = exclude_core_list + + # reduce inventory by exclude + if 'available' not in core_state: + core_state['available'] = {} + for numa in core_state['inventory'].keys(): + numa_available = [x for x in core_state['inventory'][numa] if x not in core_state['exclude']] + core_state['available'][numa] = numa_available + + if 'assignments' not in core_state: + core_state['assignments'] = {} + + # walk the nodes, consuming inventory or discovering previous allocations + # address the case where previous != desired - delete previous, re-run + for node in nodes: + + flavor = node['bmhLabels']['airshipit.org/k8s-role'] + vcpus = flavors[flavor]['vcpus'] + + for num_node in range(0, node['count']): + + # generate a unique name such as master-0, master-1 + node_name = node['name'] + '-' + str(num_node) + + # extract the core count + core_count = int(vcpus) + + # discover any previous allocation + if 'assignments' in core_state: + if node_name in core_state['assignments']: + if len(core_state['assignments'][node_name]) == core_count: + continue + else: + # TODO: support releasing the cores and adding them back + # to available + raise Exception("Existing assignment exists for node %s but does not match current core count needed" % node_name) + + # allocate the cores + allocated=False + for numa in core_state['available']: + if core_count <= len(core_state['available'][numa]): + allocated=True + cores_to_use = core_state['available'][numa][:core_count] + core_state['assignments'][node_name] = cores_to_use + core_state['available'][numa] = core_state['available'][numa][core_count:] + break + else: + continue + if not allocated: + raise Exception("Unable to find sufficient cores (%s) for node %s (available was %r)" % (core_count, node_name, core_state['available'])) + + # return a dict of nodes: cores + # or error if insufficient + with open('/etc/libvirt/vino-cores.json', 'w') as f: + f.write(json.dumps(core_state)) + + return core_state['assignments'] + + +def main(): + module = AnsibleModule( + argument_spec=dict( + nodes=dict(required=True, type='list'), + flavors=dict(required=True, type='dict'), + exclude_cpu=dict(required=True, type='str') + ) + ) + result = allocate_cores(module.params["nodes"], + module.params["flavors"], + module.params["exclude_cpu"]) + module.exit_json(**result) + +# see http://docs.ansible.com/developing_modules.html#common-module-boilerplate +from ansible.module_utils.basic import AnsibleModule # noqa + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/vino-builder/assets/playbooks/roles/libvirt/tasks/create-domain.yaml b/vino-builder/assets/playbooks/roles/libvirt/tasks/create-domain.yaml new file mode 100644 index 0000000..f6768f5 --- /dev/null +++ b/vino-builder/assets/playbooks/roles/libvirt/tasks/create-domain.yaml @@ -0,0 +1,52 @@ +- name: debug print loop + debug: + msg: "outer item={{ node }} inner item={{item}}" + loop: "{{ range(0,node.count)|list }}" + +- name: debug print virsh xml domain + debug: + msg: "{{ flavorTemplates[node['bmhLabels']['airshipit.org/k8s-role']]['domainTemplate'] }}" + loop: "{{ range(0,node.count)|list }}" + +- name: get state of existing volumes + shell: | + virsh vol-list vino-default + register: vol_list + +- name: write out domain volume request xml + copy: content="{{ flavorTemplates[node['bmhLabels']['airshipit.org/k8s-role']]['volumeTemplate'] }}" dest=/tmp/vol-{{item}}.xml + loop: "{{ range(0,node.count)|list }}" + +- name: create domain volume if it doesn't exist + shell: | + virsh vol-create vino-default /tmp/vol-{{item}}.xml + loop: "{{ range(0,node.count)|list }}" + when: "node.name + '-' + item|string not in vol_list.stdout" + +- name: ensure vino instance state directory exists + file: + path: /var/lib/libvirt/vino-instances + state: directory + recurse: yes + owner: root + group: root + +# the virt community plugin does not handle pushing out updates +# to domains, so we must shell out here instead + +- name: write out domain volume request xml + copy: content="{{ flavorTemplates[node['bmhLabels']['airshipit.org/k8s-role']]['domainTemplate'] }}" dest=/tmp/domain-{{item}}.xml + loop: "{{ range(0,node.count)|list }}" + +- name: virsh define domain + shell: | + virsh define /tmp/domain-{{item}}.xml + loop: "{{ range(0,node.count)|list }}" + +#- name: set vm to running +# virt: +# name: "{{ node.name + '-' + item|string}}" +# state: running +# autostart: yes +# loop: "{{ range(0,node.count)|list }}" +# ignore_errors: true diff --git a/vino-builder/assets/playbooks/roles/libvirt/tasks/create-network.yaml b/vino-builder/assets/playbooks/roles/libvirt/tasks/create-network.yaml new file mode 100644 index 0000000..d3e7b3d --- /dev/null +++ b/vino-builder/assets/playbooks/roles/libvirt/tasks/create-network.yaml @@ -0,0 +1,36 @@ +# Facts will be available as 'ansible_libvirt_networks' +- name: initially gather facts on existing virsh networks + virt_net: + command: facts + name: "" # this attribute is not needed but required + uri: "{{ libvirt_uri }}" + ignore_errors: true + +- name: Print value of ansible networks + debug: + msg: "Value of ansible_libvirt_networks is {{ ansible_libvirt_networks }}" + +# TODO(alanmeadows): deal with updates as once its defined we will +# never re-define it +- name: add networks defined if they do not already exist + virt_net: + state: present + # looks like setting name here is a redundant, the name is anyways taken from the template xml file, but should set it to make virt_pool module happy. + name: "{{ item.name }}" + xml: "{{ item.libvirtTemplate }}" + uri: "{{ libvirt_uri }}" + vars: + nodebridgegw: ipam.bridge_ip + +- name: activate the network + virt_net: + state: active + name: "{{ item.name }}" + uri: "{{ libvirt_uri }}" + +# these are idempotent so require no conditional checks +- name: autostart the network + virt_net: + autostart: yes + name: "{{ item.name }}" + uri: "{{ libvirt_uri }}" diff --git a/vino-builder/assets/playbooks/roles/libvirt/tasks/create-storage.yaml b/vino-builder/assets/playbooks/roles/libvirt/tasks/create-storage.yaml new file mode 100644 index 0000000..64dfd23 --- /dev/null +++ b/vino-builder/assets/playbooks/roles/libvirt/tasks/create-storage.yaml @@ -0,0 +1,18 @@ +# Facts will be available as 'ansible_libvirt_pools' +- name: initially gather facts on existing virsh pool + virt_pool: + command: facts + uri: "{{ libvirt_uri }}" + +- name: define storage the storage pool + virt_pool: + state: present + name: "{{ item.name }}" + uri: "{{ libvirt_uri }}" + xml: "{{item.libvirtTemplate}}" + +- name: activate the storage pool + virt_pool: + state: active + name: "{{ item.name }}" + uri: "{{ libvirt_uri }}" diff --git a/vino-builder/assets/playbooks/roles/libvirt/tasks/main.yml b/vino-builder/assets/playbooks/roles/libvirt/tasks/main.yml new file mode 100644 index 0000000..7e28516 --- /dev/null +++ b/vino-builder/assets/playbooks/roles/libvirt/tasks/main.yml @@ -0,0 +1,39 @@ +########################################## +# configure storage # +########################################## + +- name: create storage + include_tasks: create-storage.yaml + loop: "{{ libvirtStorage }}" + +########################################## +# configure networks # +########################################## + +# - name: create network +# include_tasks: create-network.yaml +# loop: "{{ libvirtNetworks }}" + +########################################## +# configure domains # +########################################## + +- name: allocate domain cores + core_allocation: + nodes: "{{ nodes }}" + flavors: "{{ flavors }}" + exclude_cpu: "{{ configuration.cpuExclude }}" + register: node_core_map + when: nodes + +- name: debug print node_core_map + debug: + msg: "node_core_map = {{ node_core_map }}" + +- name: define domain outer loop + include_tasks: create-domain.yaml + loop: "{{ nodes }}" + loop_control: + loop_var: node + + diff --git a/vino-builder/assets/playbooks/sample-vino-ansible-input.yaml b/vino-builder/assets/playbooks/sample-vino-ansible-input.yaml new file mode 100644 index 0000000..a73bc71 --- /dev/null +++ b/vino-builder/assets/playbooks/sample-vino-ansible-input.yaml @@ -0,0 +1,198 @@ +configuration: + cpuExclude: 0-1,54-60 + redfishCredentialSecret: + name: redfishSecret + namespace: airship-system +networks: + - name: management + subnet: 192.168.2.0/20 + allocationStart: 192.168.2.10 + allocationStop: 192.168.2.14 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc) + routes: + - to: 10.0.0.0/24 + via: "{{ ipam.bridge_ip | default(omit) }}" # vino will need to populate this from the nodelabel value `airshipit.org/vino.nodebridgegw` + dns_servers: ["135.188.34.124"] + - name: mobility-gn + subnet: 169.0.0.0/24 + routes: + - to: 0.0.0.0/0 + via: 169.0.0.1 + allocationStart: 169.0.0.10 + allocationStop: 169.0.0.254 +libvirtNetworks: + - name: management + libvirtTemplate: | + + management + + + + + + + + + + +# - name: mobility-gn +# libvirtTemplate: +libvirtStorage: + - name: vino-default + libvirtTemplate: | + + vino-default + + /var/lib/libvirt/vino + + 0711 + 0 + 0 + + + +libvirtDomains: + master: + volumeTemplate: | + {% set nodename = node.name + '-' + item|string %} + + {{ nodename }} + 0 + {{ node.instance.rootSize }} + + domainTemplate: | + {% set nodename = node.name + '-' + item|string %} + + {{ nodename }} + {{ nodename | hash('md5') }} + + {% for flavor in node.labels %} + {% for key in flavor.keys() %} + {% if key == 'vm-flavor' %} + {{ flavor[key] }} + {% endif %} + {% endfor %} + {% endfor %} + {{ ansible_date_time.date }} + + {{ node.instance.memory }} + {% if node.instance.hugepages %} + + + + + {% endif %} + {{ node.instance.vcpu }} + # function to produce list of cpus, in same numa (controled by bool), state will need to be tracked via file on hypervisor host. gotpl psudo: + + 8192 + {% for core in node_core_map[nodename] %} + + {% endfor %} + + + + /machine + + + hvm + + + + + + + + + + + + + destroy + restart + destroy + + /usr/bin/qemu-system-x86_64 + + # for each disk requested + + + + + + + + +
+ + + + + + +
+ + + # for each interface defined in vino, e.g. + + + + +
+ + + + + + + + + + + + + + + + + + + +
+ + + + + +42424:+104 + + + worker-standard: + libvirtTemplate: ... +nodes: + - name: master + labels: + - vm-flavor: master + instance: + memory: 8 + vcpu: 2 + hugepages: true + rootSize: 30 + count: 2 + BMHNetworkTemplate: + name: configMapFooThatsGoTplForNetwork + namespace: foo + field: bmhnetwork + - name: worker-standard + labels: + - vm-flavor: worker-standard + instance: + memory: 8 + vcpu: 2 + hugepages: true + rootSize: 30 + count: 0 + libvirtTemplate: | + foobar + BMHNetworkTemplate: + name: configMapFooThatsGoTplForNetwork + namespace: foo + field: bmhnetwork \ No newline at end of file diff --git a/vino-builder/assets/playbooks/vino-builder.yaml b/vino-builder/assets/playbooks/vino-builder.yaml new file mode 100644 index 0000000..6ba068c --- /dev/null +++ b/vino-builder/assets/playbooks/vino-builder.yaml @@ -0,0 +1,46 @@ + +# Licensed under the Apache License, Version 2.0 (the "License"); + +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + # - host-annotator that populates the k8s node object with approprite annotations + # - report back information such as: + # - vminfra-bridge ip address as label to k8s node + # - sushy-tools ip endpoint for BMC control + # - vino-builder (ansible) that that consumes the `ConfigMap` that contains everything necessary for libvirt to define the virtual machines and networks on the host and does both green-field generation of VM resources and understands if the `ConfigMap` changed and will handle those lifecycle updates. There is no need to stage or coordinate changes to these `ConfigMap` resources as they will result in a no-op `virsh update` which only take effect with a VM stop/start. + # - do the following (assumption is all of this is idempotent for day 2): + # - interogate host + # - prevalidate (is kvm loaded, etc) + # - define host facts (eg cpu list, vf list, etc) + # - interogate existing vms or state recording somewhere + # - collect resources in use + # - what cores are in use + # - what vfs are in use + # - memory in use + # - define libvirt networks + # - define libvirt storage pools + # - ensure appropriate qcows exist + # - define libvirt domains + # - ensure mem/cpu aligned in one numa + # - new domain validation (only on new domains): + # - do a simple domain start/destroy test via redfish. + # - wait for dhcp req on admin interface? + +--- +- hosts: localhost + + tasks: + + # generate libvirt definitions for storage, networks, and domains + - name: process libvirt definitions + include_role: + name: libvirt \ No newline at end of file