feat: add victoria logs

component-constructor.yaml

@@ -164,3 +164,27 @@ components:
           type: ociArtifact
           imageReference: "${KUSTOMIZATIONS_LOCATION_PREFIX}/metrics:${OBSERVABILITY_STACK_VERSION}"
 
+      # observability gateway
+      - name: observability-gateway-kustomization
+        version: ${OBSERVABILITY_STACK_VERSION}
+        type: kustomization
+        access:
+          type: ociArtifact
+          imageReference: "${KUSTOMIZATIONS_LOCATION_PREFIX}/observability-gateway:${OBSERVABILITY_STACK_VERSION}"
+
+      # victoria logs
+      - name: victoria-logs-kustomization
+        version: ${OBSERVABILITY_STACK_VERSION}
+        type: kustomization
+        access:
+          type: ociArtifact
+          imageReference: "${KUSTOMIZATIONS_LOCATION_PREFIX}/victoria-logs:${OBSERVABILITY_STACK_VERSION}"
+
+      - name: victoria-logs-image
+        version: ${VICTORIA_LOGS_IMAGE_VERSION}
+        type: ociImage
+        input:
+          type: ociImage
+          path: "docker.io/victoriametrics/victoria-logs:${VICTORIA_LOGS_IMAGE_VERSION}"
+          repository: images/victoria-logs
+

component-settings.yaml

@@ -26,6 +26,9 @@ PROMETHEUS_IMAGE_VERSION: "v3.10.0"
 # prometheus alertmanager
 ALERTMANAGER_IMAGE_VERSION: "v0.31.1"
 
+# victoria logs
+VICTORIA_LOGS_IMAGE_VERSION: "v1.6.0-victorialogs"
+
 
 # E2E Test dependencies
 # Not used for deployment

hack/build-component.py

@@ -75,6 +75,7 @@ def push_kustomizations(repo_root: Path, version: str) -> None:
         ("prometheus-operator", "prometheus-operator"),
         ("prometheus", "prometheus"),
         ("metrics", "metrics"),
+        ("victoria-logs", "victoria-logs")
     ]
 
     # Get git information

kustomizations/observability-gateway/gateway.yaml

@@ -0,0 +1,65 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: observability-gateway
+spec:
+  gatewayClassName: envoy-gateway
+  listeners:
+  - name: prometheus
+    port: 8443
+    protocol: HTTPS
+    hostname: "<prometheus-dnsname>"
+    allowedRoutes:
+      namespaces:
+        from: All
+    tls:
+      mode: Terminate
+      certificateRefs:
+      - kind: Secret
+        name: prometheus-cert
+  - name: victoria-logs
+    port: 8443
+    protocol: HTTPS
+    hostname: "<victoria-logs-dnsname>"
+    allowedRoutes:
+      namespaces:
+        from: All
+    tls:
+      mode: Terminate
+      certificateRefs:
+      - kind: Secret
+        name: victoria-logs-cert
+---
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: ClientTrafficPolicy
+metadata:
+  name: prometheus-mtls
+spec:
+  targetRefs:
+  - group: gateway.networking.k8s.io
+    kind: Gateway
+    name: observability-gateway
+    sectionName: prometheus
+  tls:
+    clientValidation:
+      caCertificateRefs:
+      - kind: "Secret"
+        group: ""
+        name: "prometheus-client-ca-cert"
+---
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: ClientTrafficPolicy
+metadata:
+  name: victoria-logs-mtls
+spec:
+  targetRefs:
+  - group: gateway.networking.k8s.io
+    kind: Gateway
+    name: observability-gateway
+    sectionName: victoria-logs
+  tls:
+    clientValidation:
+      caCertificateRefs:
+      - kind: "Secret"
+        group: ""
+        name: "victoria-logs-client-ca-cert"

kustomizations/observability-gateway/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - gateway.yaml
+  - gateway-issuer.yaml
+  - prometheus-certificates.yaml
+  - victoria-logs-certificates.yaml

kustomizations/prometheus/client-certificates.yaml → kustomizations/observability-gateway/prometheus-certificates.yaml

@@ -1,5 +1,17 @@
 apiVersion: cert-manager.io/v1
 kind: Certificate
+metadata:
+  name: prometheus-gateway-cert
+spec:
+  secretName: prometheus-cert
+  issuerRef:
+    name: gateway-selfsigned-issuer
+    kind: Issuer
+  dnsNames:
+  - "<dnsname>"
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
 metadata:
   name: prometheus-client-ca
 spec:

kustomizations/observability-gateway/victoria-logs-certificates.yaml

@@ -0,0 +1,47 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: victoria-logs-gateway-cert
+spec:
+  secretName: victoria-logs-cert
+  issuerRef:
+    name: gateway-selfsigned-issuer
+    kind: Issuer
+  dnsNames:
+  - "<dnsname>"
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: victoria-logs-client-ca
+spec:
+  isCA: true
+  commonName: victoria-logs-client-ca
+  secretName: victoria-logs-client-ca-cert
+  privateKey:
+    algorithm: RSA
+    size: 2048
+  issuerRef:
+    name: gateway-selfsigned-issuer
+    kind: Issuer
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: victoria-logs-client-issuer
+spec:
+  ca:
+    secretName: victoria-logs-client-ca-cert
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: victoria-logs-client-cert
+spec:
+  secretName: victoria-logs-client-cert
+  commonName: victoria-logs-client
+  usages:
+  - client auth
+  issuerRef:
+    name: victoria-logs-client-issuer
+    kind: Issuer

kustomizations/opentelemetry-collector/kustomization.yaml

@@ -2,4 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - collector.yaml
+  - log-collector.yaml
   - servicemonitor.yaml

kustomizations/opentelemetry-collector/log-collector.yaml

@@ -0,0 +1,111 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: open-telemetry-log-collector
+---
+apiVersion: opentelemetry.io/v1beta1
+kind: OpenTelemetryCollector
+metadata:
+  name: logs
+spec:
+  mode: daemonset
+  serviceAccount: open-telemetry-log-collector
+  securityContext:
+    runAsUser: 0
+  config:
+    receivers:
+      filelog:
+        include:
+          - /var/log/pods/*/*/*.log
+        start_at: beginning
+        include_file_path: true
+        include_file_name: false
+        operators:
+          # Route to the correct parser based on container runtime format
+          - type: router
+            id: get-format
+            routes:
+              - output: parser-docker
+                expr: 'body matches "^\\{"'
+            default: parser-containerd
+
+          # Docker JSON format (e.g. Docker Desktop, older clusters)
+          - type: json_parser
+            id: parser-docker
+            output: move-log-to-body
+            timestamp:
+              parse_from: attributes.time
+              layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+
+          # Containerd / CRI-O space-delimited format (most modern clusters)
+          - type: regex_parser
+            id: parser-containerd
+            regex: '^(?P<time>[^ Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$'
+            output: move-log-to-body
+            timestamp:
+              parse_from: attributes.time
+              layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+
+          # Move the parsed log field to body (both Docker and containerd / CRI-O)
+          - type: move
+            id: move-log-to-body
+            from: attributes.log
+            to: body
+            output: extract-pod-metadata
+
+          # Extract pod metadata (namespace, pod name, uid, container) from the file path
+          - type: regex_parser
+            id: extract-pod-metadata
+            parse_from: attributes["log.file.path"]
+            regex: '^/var/log/pods/(?P<k8s_namespace>[^_]+)_(?P<k8s_pod>[^_]+)_(?P<k8s_uid>[^/]+)/(?P<k8s_container>[^/]+)/\d+\.log$'
+            cache:
+              size: 128
+
+          # If the log body is JSON, promote its fields to attributes so Victoria Logs
+          # indexes them as individual fields. parse_to: attributes leaves body (= _msg)
+          # unchanged, so the original JSON string is always retained.
+          - type: json_parser
+            id: parse-json-body
+            parse_from: body
+            parse_to: attributes
+            if: 'body matches "^\\s*\\{"'
+            on_error: send
+
+    processors:
+      memory_limiter:
+        check_interval: 1s
+        limit_percentage: 75
+        spike_limit_percentage: 15
+      # Flatten nested attribute maps produced by json_parser so that nested
+      # objects (e.g. Cluster: {name, namespace}) become dot-separated scalar
+      # attributes (Cluster.name, Cluster.namespace) instead of KeyValueList
+      # values, which Victoria Logs serialises in an ugly AnyValue representation.
+      transform:
+        log_statements:
+          - context: log
+            statements:
+              - flatten(attributes, "", 10)
+      batch: {}
+
+    exporters:
+      otlphttp:
+        endpoint: "http://victoria-logs.victoria-logs-system.svc.cluster.local:9428/insert/opentelemetry"
+        tls:
+          insecure: true
+
+    service:
+      pipelines:
+        logs:
+          receivers: [filelog]
+          processors: [memory_limiter, transform, batch]
+          exporters: [otlphttp]
+
+  volumeMounts:
+    - name: varlogpods
+      mountPath: /var/log/pods
+      readOnly: true
+
+  volumes:
+    - name: varlogpods
+      hostPath:
+        path: /var/log/pods

kustomizations/prometheus/httproute.yaml

@@ -4,8 +4,9 @@ metadata:
   name: prometheus
 spec:
   parentRefs:
-  - name: prometheus
-    namespace: prometheus-system
+  - name: observability-gateway
+    namespace: observability-gateway-system
+    sectionName: prometheus
   hostnames: []
   rules:
   - matches:

kustomizations/prometheus/kustomization.yaml

@@ -3,8 +3,4 @@ kind: Kustomization
 resources:
   - prometheus.yaml
   - alertmanager.yaml
-  - gateway.yaml
-  - gateway-issuer.yaml
-  - gateway-certificate.yaml
-  - client-certificates.yaml
   - httproute.yaml

kustomizations/victoria-logs/httproute.yaml

@@ -0,0 +1,18 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: victoria-logs
+spec:
+  parentRefs:
+  - name: observability-gateway
+    namespace: observability-gateway-system
+    sectionName: victoria-logs
+  hostnames: []
+  rules:
+  - matches:
+    - path:
+        type: PathPrefix
+        value: /
+    backendRefs:
+    - name: victoria-logs
+      port: 9428

kustomizations/victoria-logs/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - victoria-logs.yaml
+  - httproute.yaml