diff --git a/build/components/versions.yml b/build/components/versions.yml index 02d20930a8..56fb0766ec 100644 --- a/build/components/versions.yml +++ b/build/components/versions.yml @@ -3,7 +3,7 @@ firmware: libvirt: v10.9.0 edk2: stable202411 core: - 3p-kubevirt: v1.6.2-v12n.24 + 3p-kubevirt: feat/virt-handler-to-hostnetwork # v1.6.2-v12n.24 3p-containerized-data-importer: v1.60.3-v12n.18 distribution: 2.8.3 package: diff --git a/images/virt-artifact/werf.inc.yaml b/images/virt-artifact/werf.inc.yaml index f30560fba6..40bc10be30 100644 --- a/images/virt-artifact/werf.inc.yaml +++ b/images/virt-artifact/werf.inc.yaml @@ -9,6 +9,7 @@ image: {{ .ModuleNamePrefix }}{{ .ImageName }}-src-artifact final: false fromImage: builder/src +fromCacheVersion: "020" # TODO: DELETE ME secrets: - id: SOURCE_REPO value: {{ $.SOURCE_REPO }} diff --git a/templates/_hostnetwork_ports.tpl b/templates/_hostnetwork_ports.tpl new file mode 100644 index 0000000000..874183edfb --- /dev/null +++ b/templates/_hostnetwork_ports.tpl @@ -0,0 +1,47 @@ +{{- /* +Port constants for DaemonSets running with hostNetwork: true. + +All three DaemonSets — virt-handler, vm-route-forge, virtualization-dra — +run with hostNetwork, so every bound port is exposed on the node's network +interfaces. Ports below are chosen outside the KubeVirt live-migration range +(4135-4199) and must not overlap with other well-known services on cluster nodes. + +Port map: + + virt-handler (kube-api-rewriter runs as its sidecar): + 4135-4199 virt-handler: live-migration tunnels (KubeVirt migration range). + 4100 virt-handler: healthz and Prometheus metrics (--port flag), kube-rbac-proxy implemented natively. + 4101 virt-handler: Console server port (--console-server-port flag). + 4102 kube-api-rewriter sidecar: Prometheus metrics (MONITORING_BIND_ADDRESS), bound to pod IP. + liveness and readiness probes (/proxy/healthz, /proxy/readyz). + 4103 kube-api-rewriter sidecar: pprof (PPROF_BIND_ADDRESS), bound to pod IP, debug mode only. + 4104 kube-api-rewriter sidecar: Kubernetes API proxy (CLIENT_PROXY_PORT), + virt-handler connects here instead of the real API server. + + vm-route-forge: + 4105 vm-route-forge: liveness and readiness probes (HEALTH_PROBE_BIND_ADDRESS). + 4106 vm-route-forge: pprof (PPROF_BIND_ADDRESS), debug mode only. + + virtualization-dra: + 4107 virtualization-dra: gRPC liveness and readiness probes. + 4280 virtualization-dra: USB/IP daemon (--usbipd-port flag). +*/ -}} + +{{- /* virt-handler */ -}} +{{- define "virt_handler.migration_port_first" -}}4135{{- end -}} +{{- define "virt_handler.migration_port_last" -}}4199{{- end -}} + +{{- define "virt_handler.port" -}}4100{{- end -}} +{{- define "virt_handler.console_server_port" -}}4101{{- end -}} +{{- define "virt_handler.rewriter_healthz_port" -}}4102{{- end -}} +{{- define "virt_handler.rewriter_monitoring_port" -}}4102{{- end -}} +{{- define "virt_handler.rewriter_pprof_port" -}}4103{{- end -}} +{{- define "virt_handler.rewriter_proxy_port" -}}4104{{- end -}} + +{{- /* vm-route-forge */ -}} +{{- define "vm_route_forge.health_port" -}}4105{{- end -}} +{{- define "vm_route_forge.pprof_port" -}}4106{{- end -}} + +{{- /* virtualization-dra */ -}} +{{- define "virtualization_dra.health_port" -}}4107{{- end -}} +{{- define "virtualization_dra.usbipd_port" -}}4280{{- end -}} diff --git a/templates/kube-api-rewriter/_customize_patch_helpers.tpl b/templates/kube-api-rewriter/_customize_patch_helpers.tpl index 72b1d18bbd..ad361d8d3e 100644 --- a/templates/kube-api-rewriter/_customize_patch_helpers.tpl +++ b/templates/kube-api-rewriter/_customize_patch_helpers.tpl @@ -30,7 +30,7 @@ spec: {{- include "kube_api_rewriter.sidecar_container" (tuple $ctx $settings) | nindent 6 }} - name: {{ $mainContainerName }} env: - {{- include "kube_api_rewriter.kubeconfig_env" . | nindent 8 }} + {{- include "kube_api_rewriter.kubeconfig_env" (tuple $ctx $settings) | nindent 8 }} volumeMounts: {{- include "kube_api_rewriter.kubeconfig_volume_mount" . | nindent 8 }} {{- end -}} diff --git a/templates/kube-api-rewriter/_settings.tpl b/templates/kube-api-rewriter/_settings.tpl index 8f54135195..f9703033a3 100644 --- a/templates/kube-api-rewriter/_settings.tpl +++ b/templates/kube-api-rewriter/_settings.tpl @@ -7,13 +7,11 @@ {{- define "kube_api_rewriter.pprof_port" -}}8129{{- end -}} +{{- define "kube_api_rewriter.client_proxy_port" -}}23915{{- end -}} + {{- define "kube_api_rewriter.env" -}} - name: LOG_LEVEL value: {{ include "moduleLogLevel" . }} -{{- if eq (include "moduleLogLevel" .) "debug" }} -- name: PPROF_BIND_ADDRESS - value: ":{{ include "kube_api_rewriter.pprof_port" . }}" -{{- end }} {{- end -}} {{- define "kube_api_rewriter.resources" -}} diff --git a/templates/kube-api-rewriter/_sidecar_helpers.tpl b/templates/kube-api-rewriter/_sidecar_helpers.tpl index 2ae379c146..ad4cf78e64 100644 --- a/templates/kube-api-rewriter/_sidecar_helpers.tpl +++ b/templates/kube-api-rewriter/_sidecar_helpers.tpl @@ -1,98 +1,34 @@ -{{- /* Helpers to add kube-api-rewriter sidecar container to a pod. - -To connect to kube-api-rewriter main controller should has KUBECONFIG env, -volumeMount with kubeconfig, and Pod should has volume with kubeconfig ConfigMap. - -These settings are provided by helpers: - -- kube_api_rewriter.kubeconfig_env defines KUBECONFIG env with file from the - mounted ConfigMap. -- kube_api_rewriter.kubeconfig_volume_mount defines volumeMount for kubeconfig ConfigMap. -- kube_api_rewriter.kubeconfig_volume defines volume with kubeconfig ConfigMap. - -Kube-api-rewriter sidecar should be the first container in the Pod, to -main controller not fail on start. - -Kube-api-rewriter sidecar works in 2 modes: without webhook or with webhook rewriting. - -Sidecar without webhook is the simplest one: - -spec: - template: - spec: - containers: - {{ include "kube_api_rewriter.sidecar_container" . | nindent 8 }} - - name: main-controller - ... - env: - {{- include "kube_api_rewriter.kubeconfig_env" . | nindent 12 }} - ... - volumeMounts: - {{- include "kube_api_rewriter.kubeconfig_volume_mount" . | nindent 12 }} - ... - volumes: - {{- include "kube_api_rewriter.kubeconfig_volume" | nindent 8 }} - ... - - -Webhook mode requires additional settings: - -- WEBHOOK_ADDRESS - address of the webhook in the main controller -- WEBHOOK_CERT_FILE - path to the webhook certificate file. -- WEBHOOK_KEY_FILE - path to the webhook key file. -- webhookCertsVolumeName - name of the Pod volume with webhook certificates. -- webhookCertsMountPath - path to mount the webhook certificates. - -The assumption here is that main controller has a webhook server and -certificates are already mounted in the Pod, so kube-api-rewriter -can use certificates from that volume to impersonate the webhook server. - -Example of adding kube-api-rewriter to the Deployment: - -spec: - template: - spec: - containers: - {{- $rewriterSettings := dict }} - {{- $_ := set $rewriterSettings "WEBHOOK_ADDRESS" "https://127.0.0.1:6443" }} - {{- $_ := set $rewriterSettings "WEBHOOK_CERT_FILE" "/etc/webhook-certificates/tls.crt" }} - {{- $_ := set $rewriterSettings "WEBHOOK_KEY_FILE" "/etc/webhook-certificates/tls.key" }} - {{- $_ := set $rewriterSettings "webhookCertsVolumeName" "webhook-certs" }} - {{- $_ := set $rewriterSettings "webhookCertsMountPath" "/etc/webhook-certificates" }} - {{- include "kube_api_rewriter.sidecar_container" (tuple . $rewriterSettings) | nindent 6 }} - - name: main-controller - ... - env: - {{- include "kube_api_rewriter.kubeconfig_env" . | nindent 12 }} - ... - ports: - - containerPort: 6443 # Goes to the WEBHOOK_ADDRESS - name: webhooks - protocol: TCP - volumeMounts: - {{- include "kube_api_rewriter.kubeconfig_volume_mount" . | nindent 12 }} - - name: webhook-certs - mountPath: /etc/webhook-certificates # Goes to the webhookCertsMountPath - readOnly: true - ... - volumes: - {{- include "kube_api_rewriter.kubeconfig_volume" | nindent 8 }} - - name: webhook-certs # Name of the existing volume goes to the webhookCertsVolumeName. - secret: - optional: true - secretName: webhook-certs - ... - - */ -}} - +{{- /* +Helpers to add the kube-api-rewriter sidecar to a Pod. + +The main container must use kube-api-rewriter via a kubeconfig mounted from the +ConfigMap exposed by these helpers: +- kube_api_rewriter.kubeconfig_env +- kube_api_rewriter.kubeconfig_volume_mount +- kube_api_rewriter.kubeconfig_volume + +The sidecar supports two modes: +- plain API proxying +- webhook rewriting, when WEBHOOK_* settings and certificate mounts are passed + +The sidecar should be placed before the main container, so the main container +does not start before the local API proxy is ready. +*/ -}} {{- define "kube_api_rewriter.image" -}} {{- include "helm_lib_module_image" (list . "kubeApiRewriter") | toJson -}} {{- end -}} - +{{- /* KUBECONFIG for the main container pointing to the local kube-api-rewriter proxy. */ -}} {{- define "kube_api_rewriter.kubeconfig_env" -}} +{{- $settings := dict -}} +{{- if (kindIs "slice" .) -}} +{{- if ge (len .) 2 -}} +{{- $settings = index . 1 -}} +{{- end -}} +{{- end -}} +{{- $kubeconfigFilename := $settings.kubeconfigFilename | default "kube-api-rewriter.kubeconfig" -}} - name: KUBECONFIG - value: /kubeconfig.local/kube-api-rewriter.kubeconfig + value: /kubeconfig.local/{{ $kubeconfigFilename }} {{- end }} {{- define "kube_api_rewriter.kubeconfig_volume" -}} @@ -107,7 +43,6 @@ spec: mountPath: /kubeconfig.local {{- end }} - {{- define "kube_api_rewriter.webhook_volume_mount" -}} {{- $volumeName := index . 0 -}} {{- $mountPath := index . 1 -}} @@ -122,16 +57,20 @@ spec: protocol: TCP {{- end }} -{{- /* Container port for the pprof server */ -}} +{{- /* Container port for the pprof server. */ -}} {{- define "kube_api_rewriter.pprof_container_port" -}} - containerPort: {{ include "kube_api_rewriter.pprof_port" . }} name: pprof protocol: TCP {{- end }} -{{- /* Sidecar container spec with kube-api-rewriter */ -}} -{{- /* Usage without the webhook proxy: {{ include kube_api_rewriter.sidecar_container . }} */ -}} -{{- /* Usage with the webhook: {{ include kube_api_rewriter.sidecar_container (tuple . $webhookSettings) }} */ -}} +{{- /* +Sidecar container spec with kube-api-rewriter. + +Usage: +- {{ include "kube_api_rewriter.sidecar_container" . }} +- {{ include "kube_api_rewriter.sidecar_container" (tuple . $settings) }} +*/ -}} {{- define "kube_api_rewriter.sidecar_container" -}} {{- $ctx := . -}} {{- $settings := dict -}} @@ -142,6 +81,15 @@ spec: {{- end -}} {{- end -}} {{- $isWebhook := hasKey $settings "WEBHOOK_ADDRESS" -}} + {{- $injectPodIP := $settings.injectPodIP | default false -}} + {{- $healthzPort := $settings.healthzPort | default 8082 -}} + {{- $healthzPath := $settings.healthzPath | default "/proxy/healthz" -}} + {{- $readyzPath := $settings.readyzPath | default "/proxy/readyz" -}} + {{- $clientProxyPort := $settings.clientProxyPort | default (include "kube_api_rewriter.client_proxy_port" $ctx | int) -}} + {{- $monitoringBindAddress := $settings.monitoringBindAddress | default "127.0.0.1:9090" -}} + {{- $pprofBindAddress := $settings.pprofBindAddress | default (printf ":%s" (include "kube_api_rewriter.pprof_port" $ctx)) -}} + {{- $pprofPort := last (splitList ":" $pprofBindAddress) | int -}} + {{- $probeScheme := $settings.probeScheme | default "HTTPS" -}} - name: {{ include "kube_api_rewriter.sidecar_name" $ctx }} image: {{ include "kube_api_rewriter.image" $ctx }} imagePullPolicy: IfNotPresent @@ -154,8 +102,24 @@ spec: - name: WEBHOOK_KEY_FILE value: "{{ $settings.WEBHOOK_KEY_FILE }}" {{- end }} + {{- if $injectPodIP }} + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + {{- end }} + - name: CLIENT_PROXY_PORT + value: "{{ $clientProxyPort }}" - name: MONITORING_BIND_ADDRESS - value: "127.0.0.1:9090" + value: "{{ $monitoringBindAddress }}" + {{- if $settings.monitoringAuth }} + - name: MONITORING_AUTH + value: {{ $settings.monitoringAuth | toJson | quote }} + {{- end }} + {{- if eq (include "moduleLogLevel" $ctx) "debug" }} + - name: PPROF_BIND_ADDRESS + value: "{{ $pprofBindAddress }}" + {{- end }} {{- include "kube_api_rewriter.env" $ctx | nindent 4 }} resources: requests: @@ -173,15 +137,15 @@ spec: type: RuntimeDefault livenessProbe: httpGet: - path: /proxy/healthz - port: 8082 - scheme: HTTPS + path: {{ $healthzPath }} + port: {{ $healthzPort }} + scheme: {{ $probeScheme }} initialDelaySeconds: 10 readinessProbe: httpGet: - path: /proxy/readyz - port: 8082 - scheme: HTTPS + path: {{ $readyzPath }} + port: {{ $healthzPort }} + scheme: {{ $probeScheme }} initialDelaySeconds: 10 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File @@ -191,9 +155,13 @@ spec: {{- end }} ports: {{- if eq (include "moduleLogLevel" $ctx) "debug" }} - {{- include "kube_api_rewriter.pprof_container_port" . | nindent 4 }} + - containerPort: {{ $pprofPort }} + name: pprof + protocol: TCP {{- end }} - {{- if $isWebhook -}} - {{- include "kube_api_rewriter.webhook_container_port" .| nindent 4 }} + {{- if $isWebhook }} + - containerPort: {{ include "kube_api_rewriter.webhook_port" $ctx }} + name: {{ include "kube_api_rewriter.webhook_port_name" $ctx }} + protocol: TCP {{- end -}} {{- end -}} diff --git a/templates/kube-api-rewriter/cm-kubeconfig-local.yaml b/templates/kube-api-rewriter/cm-kubeconfig-local.yaml index 966a348e5b..bbdbb380d6 100644 --- a/templates/kube-api-rewriter/cm-kubeconfig-local.yaml +++ b/templates/kube-api-rewriter/cm-kubeconfig-local.yaml @@ -18,3 +18,4 @@ data: cluster: kube-api-rewriter name: kube-api-rewriter current-context: kube-api-rewriter + diff --git a/templates/kubevirt/_kubevirt_helpers.tpl b/templates/kubevirt/_kubevirt_helpers.tpl index 942ffb14aa..0904549932 100644 --- a/templates/kubevirt/_kubevirt_helpers.tpl +++ b/templates/kubevirt/_kubevirt_helpers.tpl @@ -63,6 +63,110 @@ spec: '{{ include "kubevirt.delve_strategic_patch" . | fromYaml | toJson }}' {{- end }} +{{- define "kubevirt.virt_handler_ports_strategic_patch" -}} +spec: + template: + spec: + containers: + - name: virt-handler + ports: + - containerPort: {{ include "virt_handler.port" . | int }} + name: metrics + protocol: TCP + - containerPort: {{ include "virt_handler.console_server_port" . | int }} + name: console + protocol: TCP +{{- end -}} + +{{- define "kubevirt.virt_handler_ports_strategic_patch_json" -}} +'{{ include "kubevirt.virt_handler_ports_strategic_patch" . | fromYaml | toJson }}' +{{- end }} + +{{- define "kubevirt.virt_api_args_strategic_patch" -}} +spec: + template: + spec: + containers: + - name: virt-api + args: + - --port + - "8443" + - --console-server-port + - {{ include "virt_handler.console_server_port" . | quote }} + - --subresources-only + - -v + - "2" +{{- end -}} + +{{- define "kubevirt.virt_api_args_strategic_patch_json" -}} +'{{ include "kubevirt.virt_api_args_strategic_patch" . | fromYaml | toJson }}' +{{- end }} + +{{- define "kubevirt.virt_handler_args_strategic_patch" -}} +spec: + template: + spec: + containers: + - name: virt-handler + args: + - --port + - {{ include "virt_handler.port" . | quote }} + - --hostname-override + - $(NODE_NAME) + - --pod-ip-address + - $(MY_POD_IP) + - --max-metric-requests + - "3" + - --console-server-port + - {{ include "virt_handler.console_server_port" . | quote }} + - --migration-port-range-enabled + - "true" + - --migration-port-range-first + - {{ include "virt_handler.migration_port_first" . | quote }} + - --migration-port-range-last + - {{ include "virt_handler.migration_port_last" . | quote }} + - --graceful-shutdown-seconds + - "315" + - -v + - "2" +{{- end -}} + +{{- define "kubevirt.virt_handler_args_strategic_patch_json" -}} +'{{ include "kubevirt.virt_handler_args_strategic_patch" . | fromYaml | toJson }}' +{{- end }} + +{{- define "kubevirt.virt_handler_probes_strategic_patch" -}} +spec: + template: + spec: + containers: + - name: virt-handler + livenessProbe: + httpGet: + path: /healthz + port: {{ include "virt_handler.port" . | int }} + scheme: HTTPS + failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 45 + successThreshold: 1 + timeoutSeconds: 10 + readinessProbe: + httpGet: + path: /healthz + port: {{ include "virt_handler.port" . | int }} + scheme: HTTPS + failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 20 + successThreshold: 1 + timeoutSeconds: 10 +{{- end -}} + +{{- define "kubevirt.virt_handler_probes_strategic_patch_json" -}} +'{{ include "kubevirt.virt_handler_probes_strategic_patch" . | fromYaml | toJson }}' +{{- end }} + {{/* Calculate parallel migrations per cluster. This template returns: - Count of nodes with virt-handler if kubevirt config is in 'Deployed' phase. diff --git a/templates/kubevirt/kubevirt.yaml b/templates/kubevirt/kubevirt.yaml index 55ad3fcd6c..7c7ef87997 100644 --- a/templates/kubevirt/kubevirt.yaml +++ b/templates/kubevirt/kubevirt.yaml @@ -75,22 +75,6 @@ spec: virtualMachineOptions: disableSerialConsoleLog: {} customizeComponents: - flags: - {{- if ne "delve/virt-api" ($delve | dig "debug" "component" "") }} - api: - metrics-listen: 127.0.0.1 - metrics-port: "8080" - {{- end }} - {{- if ne "delve/virt-controller" ($delve | dig "debug" "component" "") }} - controller: - metrics-listen: 127.0.0.1 - metrics-port: "8080" - {{- end }} - {{- if ne "delve/virt-handler" ($delve | dig "debug" "component" "") }} - handler: - metrics-listen: 127.0.0.1 - metrics-port: "8080" - {{- end }} patches: # Add node placement settings for virt-api, virt-controller, virt-operator, virt-handler. - resourceType: Deployment @@ -113,6 +97,10 @@ spec: resourceName: virt-handler patch: '[{"op":"replace","path":"/spec/template/spec/tolerations","value":{{ $tolerationsAnyNode }}}]' type: json + - resourceType: DaemonSet + resourceName: virt-handler + patch: '[{"op":"replace","path":"/spec/template/spec/hostNetwork","value":true}]' + type: json {{- if and $delve (hasKey $delve "debug") }} # Debug {{- if eq $delve.debug.component "delve/virt-api" }} @@ -175,67 +163,25 @@ spec: type: strategic {{- end }} - # Add kube-api-rewriter sidecar containers to virt-controller, virt-api, virt-handler and virt-exportproxy. - - resourceName: virt-controller - resourceType: Deployment - patch: {{ include "kube_api_rewriter.pod_spec_strategic_patch_json" (list . "virt-controller") }} - type: strategic - + # Add kube-api-rewriter sidecar containers to virt-api and virt-exportproxy. {{- $virtApiRewriterSettings := dict }} {{- $_ := set $virtApiRewriterSettings "WEBHOOK_ADDRESS" "https://127.0.0.1:8443" }} {{- $_ := set $virtApiRewriterSettings "WEBHOOK_CERT_FILE" "/etc/virt-api/certificates/tls.crt" }} {{- $_ := set $virtApiRewriterSettings "WEBHOOK_KEY_FILE" "/etc/virt-api/certificates/tls.key" }} {{- $_ := set $virtApiRewriterSettings "webhookCertsVolumeName" "kubevirt-virt-api-certs" }} {{- $_ := set $virtApiRewriterSettings "webhookCertsMountPath" "/etc/virt-api/certificates" }} + {{- $_ := set $virtApiRewriterSettings "healthzPath" "/healthz" }} + {{- $_ := set $virtApiRewriterSettings "readyzPath" "/readyz" }} + {{- $_ := set $virtApiRewriterSettings "healthzPort" 9090 }} + {{- $_ := set $virtApiRewriterSettings "probeScheme" "HTTP" }} + {{- $_ := set $virtApiRewriterSettings "injectPodIP" true }} + {{- $_ := set $virtApiRewriterSettings "monitoringBindAddress" "$(POD_IP):9090" }} + {{- $_ := set $virtApiRewriterSettings "monitoringAuth" (dict "group" "apps" "version" "v1" "resource" "deployments" "namespace" (printf "d8-%s" .Chart.Name) "name" "virt-api" "subresource" "prometheus-metrics") }} - resourceName: virt-api resourceType: Deployment patch: {{ include "kube_api_rewriter.pod_spec_strategic_patch_json" (tuple . "virt-api" $virtApiRewriterSettings) }} type: strategic - - resourceName: virt-handler - resourceType: DaemonSet - patch: {{ include "kube_api_rewriter.pod_spec_strategic_patch_json" (list . "virt-handler") }} - type: strategic - - # Add kube-api-rewriter sidecar containers to virt-controller, virt-api, virt-handler. - {{- $kubeRbacProxySettings := dict }} - {{- $_ := set $kubeRbacProxySettings "runAsUserNobody" true }} - {{- $_ := set $kubeRbacProxySettings "ignorePaths" "/proxy/healthz,/proxy/readyz" }} - {{- $_ := set $kubeRbacProxySettings "upstreams" (list - (dict "upstream" "http://127.0.0.1:9090/metrics" "path" "/proxy/metrics" "name" "kube-api-rewriter") - (dict "upstream" "http://127.0.0.1:8080/metrics" "path" "/metrics" "name" "virt-controller") - (dict "upstream" "http://127.0.0.1:9090/healthz" "path" "/proxy/healthz" "name" "kube-api-rewriter") - (dict "upstream" "http://127.0.0.1:9090/readyz" "path" "/proxy/readyz" "name" "kube-api-rewriter") - ) }} - - resourceName: virt-controller - resourceType: Deployment - patch: {{ include "kube_rbac_proxy.pod_spec_strategic_patch_json" (tuple . $kubeRbacProxySettings) }} - type: strategic - - {{- $_ := set $kubeRbacProxySettings "ignorePaths" "/proxy/healthz,/proxy/readyz" }} - {{- $_ := set $kubeRbacProxySettings "upstreams" (list - (dict "upstream" "http://127.0.0.1:9090/metrics" "path" "/proxy/metrics" "name" "kube-api-rewriter") - (dict "upstream" "http://127.0.0.1:8080/metrics" "path" "/metrics" "name" "virt-api") - (dict "upstream" "http://127.0.0.1:9090/healthz" "path" "/proxy/healthz" "name" "kube-api-rewriter") - (dict "upstream" "http://127.0.0.1:9090/readyz" "path" "/proxy/readyz" "name" "kube-api-rewriter") - ) }} - - resourceName: virt-api - resourceType: Deployment - patch: {{ include "kube_rbac_proxy.pod_spec_strategic_patch_json" (tuple . $kubeRbacProxySettings) }} - type: strategic - - {{- $_ := set $kubeRbacProxySettings "ignorePaths" "/proxy/healthz,/proxy/readyz" }} - {{- $_ := set $kubeRbacProxySettings "upstreams" (list - (dict "upstream" "http://127.0.0.1:9090/metrics" "path" "/proxy/metrics" "resource" "daemonsets" "name" "kube-api-rewriter") - (dict "upstream" "http://127.0.0.1:8080/metrics" "path" "/metrics" "resource" "daemonsets" "name" "virt-handler") - (dict "upstream" "http://127.0.0.1:9090/healthz" "path" "/proxy/healthz" "resource" "daemonsets" "name" "kube-api-rewriter") - (dict "upstream" "http://127.0.0.1:9090/readyz" "path" "/proxy/readyz" "resource" "daemonsets" "name" "kube-api-rewriter") - ) }} - - resourceName: virt-handler - resourceType: DaemonSet - patch: {{ include "kube_rbac_proxy.pod_spec_strategic_patch_json" (tuple . $kubeRbacProxySettings) }} - type: strategic - # Add rewriter proxy container port to Services used by webhook configurations. # First need to set name for existing port to make strategic patch works later. - resourceName: virt-api @@ -330,11 +276,6 @@ spec: resourceName: virt-handler patch: {{ include "pod_spec_priority_class_name_patch" $priorityClassName }} type: strategic - # Patch service for https-metrics - - resourceType: Service - resourceName: kubevirt-prometheus-metrics - patch: '[{"op": "replace", "path": "/spec/ports/0/targetPort", "value": "https-metrics"}]' - type: json # Additional environment variables for virt-controller. {{ define "virt-controller-additional-envs" }} @@ -356,6 +297,34 @@ env: patch: '{"spec":{"template":{"metadata":{"labels":{"security.deckhouse.io/security-policy-exception": "virt-handler-ds"}}}}}' type: strategic + # Expose virt-handler ports: health API (--port) and console server (--console-server-port). + - resourceName: virt-handler + resourceType: DaemonSet + patch: {{ include "kubevirt.virt_handler_ports_strategic_patch_json" . }} + type: strategic + + # Rewrite virt-api args, replacing the default ports baked into the image. + # This is required because customizeComponents.flags only appends flags and cannot replace existing ones. + - resourceName: virt-api + resourceType: Deployment + patch: {{ include "kubevirt.virt_api_args_strategic_patch_json" . }} + type: strategic + + # Rewrite virt-handler args with hostNetwork ports, replacing the default ports baked into the image. + # This is required because customizeComponents.flags only appends flags and cannot replace existing ones. + - resourceName: virt-handler + resourceType: DaemonSet + patch: {{ include "kubevirt.virt_handler_args_strategic_patch_json" . }} + type: strategic + + # Override virt-handler liveness and readiness probes to use the new host-network port. + - resourceName: virt-handler + resourceType: DaemonSet + patch: {{ include "kubevirt.virt_handler_probes_strategic_patch_json" . }} + type: strategic + + # Change host path for directory with capabilities xml files. We have custom qemu with different + # machine types thus it conflicts with the original kubevirt. {{ define "virt-handler-rewrite-host-path-volumes"}} volumes: # Directory with capabilities xml files. We have custom qemu with different diff --git a/templates/kubevirt/virt-operator/deployment.yaml b/templates/kubevirt/virt-operator/deployment.yaml index 833ef6ccf3..2afa4e3c7d 100644 --- a/templates/kubevirt/virt-operator/deployment.yaml +++ b/templates/kubevirt/virt-operator/deployment.yaml @@ -33,7 +33,6 @@ spec: resourcePolicy: containerPolicies: {{- include "kube_api_rewriter.vpa_container_policy" . | nindent 4 }} - {{- include "kube_rbac_proxy.vpa_container_policy" . | nindent 4 }} - containerName: virt-operator minAllowed: {{- include "virt_operator_resources" . | nindent 8 }} @@ -95,26 +94,19 @@ spec: {{- $_ := set $rewriterSettings "WEBHOOK_KEY_FILE" "/etc/virt-operator/certificates/tls.key" }} {{- $_ := set $rewriterSettings "webhookCertsVolumeName" "kubevirt-operator-certs" }} {{- $_ := set $rewriterSettings "webhookCertsMountPath" "/etc/virt-operator/certificates" }} + {{- $_ := set $rewriterSettings "healthzPath" "/healthz" }} + {{- $_ := set $rewriterSettings "readyzPath" "/readyz" }} + {{- $_ := set $rewriterSettings "healthzPort" 9090 }} + {{- $_ := set $rewriterSettings "probeScheme" "HTTP" }} + {{- $_ := set $rewriterSettings "injectPodIP" true }} + {{- $_ := set $rewriterSettings "monitoringBindAddress" "$(POD_IP):9090" }} + {{- $_ := set $rewriterSettings "monitoringAuth" (dict "group" "apps" "version" "v1" "resource" "deployments" "namespace" (printf "d8-%s" .Chart.Name) "name" "virt-operator" "subresource" "prometheus-metrics") }} {{- include "kube_api_rewriter.sidecar_container" (tuple . $rewriterSettings) | nindent 6 }} - {{- $kubeRbacProxySettings := dict }} - {{- $_ := set $kubeRbacProxySettings "runAsUserNobody" true }} - {{- $_ := set $kubeRbacProxySettings "ignorePaths" "/proxy/healthz,/proxy/readyz" }} - {{- $_ := set $kubeRbacProxySettings "upstreams" (list - (dict "upstream" "http://127.0.0.1:9090/metrics" "path" "/proxy/metrics" "name" "kube-api-rewriter") - (dict "upstream" "http://127.0.0.1:8080/metrics" "path" "/metrics" "name" "virt-operator") - (dict "upstream" "http://127.0.0.1:9090/healthz" "path" "/proxy/healthz" "name" "kube-api-rewriter") - (dict "upstream" "http://127.0.0.1:9090/readyz" "path" "/proxy/readyz" "name" "kube-api-rewriter") - ) }} - {{- include "kube_rbac_proxy.sidecar_container" (tuple . $kubeRbacProxySettings) | nindent 6 }} - name: virt-operator {{- include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted" . | nindent 8 }} args: - --port - "8443" - - --metrics-listen - - 127.0.0.1 - - --metrics-port - - "8080" - -v - "2" command: @@ -136,13 +128,13 @@ spec: httpGet: path: /healthz port: 8443 - scheme: HTTP + scheme: HTTPS initialDelaySeconds: 10 livenessProbe: httpGet: path: /healthz port: 8443 - scheme: HTTP + scheme: HTTPS initialDelaySeconds: 10 resources: requests: diff --git a/templates/virtualization-dra/daemonset.yaml b/templates/virtualization-dra/daemonset.yaml index 16b04b81f0..871c2845f8 100644 --- a/templates/virtualization-dra/daemonset.yaml +++ b/templates/virtualization-dra/daemonset.yaml @@ -167,7 +167,8 @@ spec: args: - {{ include "virtualization-dra.featureGates" . }} {{/* https://github.com/deckhouse/deckhouse/pull/18139 */}} - - --usbipd-port=4280 + - --usbipd-port={{ include "virtualization_dra.usbipd_port" . }} + - --healthz-port={{ include "virtualization_dra.health_port" . }} {{- if eq (include "moduleLogLevel" .) "debug" }} - --log-level=debug - --log-debug-verbosity=10 @@ -189,23 +190,23 @@ spec: {{- include "virtualization-dra_resources" . | nindent 14 }} {{- end }} ports: - - containerPort: 4280 + - containerPort: {{ include "virtualization_dra.usbipd_port" . }} name: usbipd protocol: TCP - - containerPort: 51515 + - containerPort: {{ include "virtualization_dra.health_port" . }} name: health protocol: TCP {{- include "delvePorts" (list $delve "delve/virtualization-dra") | nindent 12 }} {{- if ne "delve/virtualization-dra" ($delve | dig "debug" "component" "") }} readinessProbe: grpc: - port: 51515 + port: {{ include "virtualization_dra.health_port" . }} service: liveness failureThreshold: 3 periodSeconds: 10 livenessProbe: grpc: - port: 51515 + port: {{ include "virtualization_dra.health_port" . }} service: liveness failureThreshold: 3 periodSeconds: 10 diff --git a/templates/vm-route-forge/daemonset.yaml b/templates/vm-route-forge/daemonset.yaml index e0816552e8..62b08cbc13 100644 --- a/templates/vm-route-forge/daemonset.yaml +++ b/templates/vm-route-forge/daemonset.yaml @@ -122,10 +122,10 @@ spec: {{- end }} {{- if eq (include "moduleLogLevel" .) "debug" }} - name: PPROF_BIND_ADDRESS - value: ":8119" + value: ":{{ include "vm_route_forge.pprof_port" . }}" {{- end }} - name: HEALTH_PROBE_BIND_ADDRESS - value: "127.0.0.1:8118" + value: "127.0.0.1:{{ include "vm_route_forge.health_port" . }}" resources: requests: {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 14 }} @@ -133,10 +133,10 @@ spec: {{- include "vm-route-forge_resources" . | nindent 14 }} {{- end }} ports: - - containerPort: 8119 + - containerPort: {{ include "vm_route_forge.pprof_port" . }} name: pprof protocol: TCP - - containerPort: 8118 + - containerPort: {{ include "vm_route_forge.health_port" . }} name: health protocol: TCP {{- include "delvePorts" (list $delve "delve/vm-route-forge") | nindent 12 }} @@ -145,7 +145,7 @@ spec: httpGet: host: localhost path: /readyz - port: 8118 + port: {{ include "vm_route_forge.health_port" . }} scheme: HTTP initialDelaySeconds: 5 failureThreshold: 2 @@ -154,7 +154,7 @@ spec: httpGet: host: localhost path: /healthz - port: 8118 + port: {{ include "vm_route_forge.health_port" . }} scheme: HTTP periodSeconds: 1 failureThreshold: 3 diff --git a/templates/vm-route-forge/service.yaml b/templates/vm-route-forge/service.yaml index e04086d5a0..df5a67f28b 100644 --- a/templates/vm-route-forge/service.yaml +++ b/templates/vm-route-forge/service.yaml @@ -9,7 +9,7 @@ metadata: spec: ports: - name: pprof - port: 8119 + port: {{ include "vm_route_forge.pprof_port" . }} protocol: TCP targetPort: pprof selector: