debian: add Docker service to helm chart

* New pod stx-docker running "docker-in-docker" image * New optional env var STX_INSECURE_DOCKER_REGISTRIES to disable SSL validation for select registries in stx-docker * stx-builder: install docker client in POD image * stx-builder: add env vars that point docker client to the new docker service POD TESTS ===== * Rebuild all build images * Make sure docker works within "builder" pod: - docker pull - docker build - docker login - docker push - docker run with volume mounts under /localdisk/{loadbuild,designer}, make sure these location are visible and can be mounted by the docker service Story: 2009897 Task: 44691 Change-Id: I2b51c5f90bf2dee1ed6a159c60d76fc05e1f325a Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
2022-02-25 14:53:26 -05:00
parent 01144d82d1
commit 411261ae81
13 changed files with 302 additions and 7 deletions
--- a/4
+++ b/4
@@ -48,6 +48,10 @@
 #     K8s namespace for builder pods & services. For vanilla k8s only.
 #     Minikube always uses the default namespace.
 #
+#   STX_INSECURE_DOCKER_REGISTRIES
+#     Space-separated list of docker registries for which we want to disable
+#     SSL certificate validation. Only affects docker running in builder pods.
+#     Requires pod restart when changed.
 #

 notice_warn () {
--- a/stx/dockerfiles/stx-builder.Dockerfile
+++ b/stx/dockerfiles/stx-builder.Dockerfile
@@ -71,7 +71,8 @@ RUN curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor
    echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \
  $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \
-  apt-get update
+  apt-get update && \
+  apt-get install --no-install-recommends -y docker-ce-cli

 COPY stx/toCOPY/lat-tool/lat /opt/LAT/lat
 COPY stx/toCOPY/builder/finishSetup.sh /usr/local/bin
--- a/stx/lib/stx/config.py
+++ b/stx/lib/stx/config.py
@@ -15,6 +15,7 @@
 #
 import logging
 import os
+import re

 from stx import stx_configparser
 from stx import utils
@@ -72,6 +73,12 @@ class Config:
        self.kubectl_cmd = None
        self.helm_cmd = None

+        reg_list_str = os.getenv('STX_INSECURE_DOCKER_REGISTRIES')
+        if reg_list_str:
+            self._insecure_docker_reg_list = re.split(r'[ \t;,]+', reg_list_str)
+        else:
+            self._insecure_docker_reg_list = []
+
    def load(self):
        """Load stx.conf"""
        self.data = stx_configparser.STXConfigParser(self.config_filename)
@@ -101,6 +108,11 @@ class Config:
        assert self.data
        return self.helm_cmd

+    @property
+    def insecure_docker_reg_list(self):
+        """List of insecure docker registries we are allowed to access"""
+        return self._insecure_docker_reg_list
+
    def _init_kubectl_cmd(self):
        # helm
        self.helm_cmd = 'helm'
--- a/stx/lib/stx/stx_control.py
+++ b/stx/lib/stx/stx_control.py
@@ -190,6 +190,9 @@ stx-pkgbuilder/configmap/')
            # need to review this to support multi node (PV/PVCs)
            cmd += ' --set global.hostDir=' + self.config.build_home

+        for reg_index, reg in enumerate(self.config.insecure_docker_reg_list):
+            cmd += f' --set stx-docker.insecureRegistries[{reg_index}]={reg}'
+
        self.logger.debug('Execute the helm start command: %s', cmd)
        helm_status = self.k8s.helm_release_exists(self.projectname)
        if helm_status:
@@ -225,7 +228,7 @@ stx-pkgbuilder/configmap/')

    def handleEnterTask(self, args):
        default_docker = 'builder'
-        container_list = ['builder', 'pkgbuilder', 'repomgr', 'lat']
+        container_list = ['builder', 'pkgbuilder', 'repomgr', 'lat', 'docker']
        prefix_exec_cmd = self.config.kubectl() + ' exec -ti '

        if args.dockername:
@@ -241,6 +244,8 @@ argument. eg: %s \n', container_list)
                cmd = prefix_exec_cmd + podname
                cmd = cmd + ' -- bash -l -c \'runuser -u ${MYUNAME} -- bash \
 --rcfile /home/$MYUNAME/userenv\''
+            elif default_docker == 'docker':
+                cmd = prefix_exec_cmd + podname + ' -- sh'
            else:
                cmd = prefix_exec_cmd + podname + ' -- bash'
            self.logger.debug('Execute the enter command: %s', cmd)
--- a/stx/lib/stx/stx_main.py
+++ b/stx/lib/stx/stx_main.py
@@ -61,11 +61,10 @@ task.\t\teg: [start|enter|stop|status|upgrade]')
                                       Upgrade the stx-builder/obs/lat/pulp \
                                       containers.\n\n')
        control_subparser.add_argument('--dockername',
-                                       help='[ builder|pkgbuilder|repomgr|lat \
-                                       ], the four dockers you can enter, if \
-                                       there is no this argument for enter \
-                                       task, default enter the stx-builder \
-                                       container\n\n', required=False)
+                                       help='[ builder|pkgbuilder|repomgr|' +
+                                       'lat|docker ]: container name to ' +
+                                       'enter, default: builder\n\n',
+                                       required=False)
        control_subparser.set_defaults(handle=self.handlecontrol.handleControl)

        config_subparser = subparsers.add_parser('config',
--- a/stx/stx-build-tools-chart/stx-builder/Chart.yaml
+++ b/stx/stx-build-tools-chart/stx-builder/Chart.yaml
@@ -33,3 +33,6 @@ dependencies:
  - name: stx-repomgr
    version: "0.1.0"
    repository: "file://dependency_chart/stx-aptly/stx-repomgr"
+  - name: stx-docker
+    version: "0.1.0"
+    repository: "file://dependency_chart/stx-docker"
--- a/stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-docker/.helmignore
+++ b/stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-docker/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
--- a/stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-docker/Chart.yaml
+++ b/stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-docker/Chart.yaml
@@ -0,0 +1,24 @@
+---
+apiVersion: v2
+name: stx-docker
+description: A Helm chart for the docker daemon in StarlingX build environment
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+appVersion: 1.16.0
--- a/stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-docker/templates/_helpers.tpl
+++ b/stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-docker/templates/_helpers.tpl
@@ -0,0 +1,63 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "stx-docker.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "stx-docker.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "stx-docker.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "stx-docker.labels" -}}
+helm.sh/chart: {{ include "stx-docker.chart" . }}
+{{ include "stx-docker.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "stx-docker.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "stx-docker.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "stx-docker.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "stx-docker.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
--- a/stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-docker/templates/deployment.yaml
+++ b/stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-docker/templates/deployment.yaml
@@ -0,0 +1,71 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "stx-docker.fullname" . }}
+  labels:
+    {{- include "stx-docker.labels" . | nindent 4 }}
+spec:
+{{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.replicaCount }}
+{{- end }}
+  selector:
+    matchLabels:
+      {{- include "stx-docker.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+    {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+      labels:
+        {{- include "stx-docker.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          tty: true
+          env:
+          - name: DOCKER_TLS_CERTDIR
+            value: ""
+          command:
+          - "/usr/local/bin/dockerd-entrypoint.sh"
+          {{- range .Values.insecureRegistries }}
+          - "--insecure-registry={{ . }}"
+          {{- end }}
+          volumeMounts:
+          - name: shared-workspace
+            mountPath: {{ .Values.volumes.sharedWorkspace.mountPath }}
+          - name: docker-run
+            mountPath: {{ .Values.volumes.dockerRun.mountPath }}
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+
+      volumes:
+      - name: shared-workspace
+        hostPath:
+          path: {{ .Values.global.hostDir }}{{ .Values.volumes.sharedWorkspace.hostPath }}
+      - name: docker-run
+        hostPath:
+          path: {{ .Values.global.hostDir }}{{ .Values.volumes.dockerRun.hostPath }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
--- a/stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-docker/templates/service.yaml
+++ b/stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-docker/templates/service.yaml
@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "stx-docker.fullname" . }}
+  labels:
+    {{- include "stx-docker.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.services.dockerDaemon.type }}
+  ports:
+    - port: {{ .Values.services.dockerDaemon.port }}
+      targetPort: 2375
+      protocol: TCP
+      name: docker-socket
+  selector:
+    {{- include "stx-repomgr.selectorLabels" . | nindent 4 }}
--- a/stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-docker/values.yaml
+++ b/stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-docker/values.yaml
@@ -0,0 +1,71 @@
+# Default values for stx-lat-tool.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+---
+replicaCount: 1
+
+image:
+  repository: docker
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: "20.10.12-dind"
+
+volumes:
+  sharedWorkspace:
+    mountPath: /localdisk
+    hostPath: /localdisk
+  dockerRun:
+    mountPath: /var/run/docker
+    hostPath: /docker/run
+
+services:
+  dockerDaemon:
+    type: ClusterIP
+    port: 2375
+
+insecureRegistries:
+#  - "registry.address:port"
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+podAnnotations: {}
+
+podSecurityContext: {}
+# fsGroup: 2000
+
+securityContext:
+  privileged: true
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+resources: {}
+# We usually recommend not to specify default resources and to leave this as a conscious
+# choice for the user. This also increases chances charts run on environments with little
+# resources, such as Minikube. If you do want to specify resources, uncomment the following
+# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+# limits:
+#   cpu: 100m
+#   memory: 128Mi
+# requests:
+#   cpu: 100m
+#   memory: 128Mi
+
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
--- a/stx/stx-build-tools-chart/stx-builder/templates/deployment.yaml
+++ b/stx/stx-build-tools-chart/stx-builder/templates/deployment.yaml
@@ -34,6 +34,9 @@ spec:
          image: "{{ .Values.image.repository }}:{{ .Values.global.image.tag | default .Chart.AppVersion }}"
          imagePullPolicy: {{ .Values.image.pullPolicy }}
          tty: true
+          env:
+          - name: DOCKER_HOST
+            value: "tcp://{{ .Release.Name }}-stx-docker"
          volumeMounts:
          - name: {{ .Values.volumeMounts.workspace.name }}
            mountPath: {{ .Values.volumeMounts.workspace.mountPath}}