diff --git a/component-constructor.yaml b/component-constructor.yaml index cf1c9e5..66dc79c 100644 --- a/component-constructor.yaml +++ b/component-constructor.yaml @@ -164,3 +164,27 @@ components: type: ociArtifact imageReference: "${KUSTOMIZATIONS_LOCATION_PREFIX}/metrics:${OBSERVABILITY_STACK_VERSION}" + # observability gateway + - name: observability-gateway-kustomization + version: ${OBSERVABILITY_STACK_VERSION} + type: kustomization + access: + type: ociArtifact + imageReference: "${KUSTOMIZATIONS_LOCATION_PREFIX}/observability-gateway:${OBSERVABILITY_STACK_VERSION}" + + # victoria logs + - name: victoria-logs-kustomization + version: ${OBSERVABILITY_STACK_VERSION} + type: kustomization + access: + type: ociArtifact + imageReference: "${KUSTOMIZATIONS_LOCATION_PREFIX}/victoria-logs:${OBSERVABILITY_STACK_VERSION}" + + - name: victoria-logs-image + version: ${VICTORIA_LOGS_IMAGE_VERSION} + type: ociImage + input: + type: ociImage + path: "docker.io/victoriametrics/victoria-logs:${VICTORIA_LOGS_IMAGE_VERSION}" + repository: images/victoria-logs + diff --git a/component-settings.yaml b/component-settings.yaml index 9a42ce6..0cdb69e 100644 --- a/component-settings.yaml +++ b/component-settings.yaml @@ -26,6 +26,9 @@ PROMETHEUS_IMAGE_VERSION: "v3.10.0" # prometheus alertmanager ALERTMANAGER_IMAGE_VERSION: "v0.31.1" +# victoria logs +VICTORIA_LOGS_IMAGE_VERSION: "v1.6.0-victorialogs" + # E2E Test dependencies # Not used for deployment diff --git a/hack/build-component.py b/hack/build-component.py index 6e20722..126d20e 100755 --- a/hack/build-component.py +++ b/hack/build-component.py @@ -75,6 +75,7 @@ def push_kustomizations(repo_root: Path, version: str) -> None: ("prometheus-operator", "prometheus-operator"), ("prometheus", "prometheus"), ("metrics", "metrics"), + ("victoria-logs", "victoria-logs") ] # Get git information diff --git a/kustomizations/prometheus/gateway-issuer.yaml b/kustomizations/observability-gateway/gateway-issuer.yaml similarity index 100% rename from kustomizations/prometheus/gateway-issuer.yaml rename to kustomizations/observability-gateway/gateway-issuer.yaml diff --git a/kustomizations/observability-gateway/gateway.yaml b/kustomizations/observability-gateway/gateway.yaml new file mode 100644 index 0000000..a927fb8 --- /dev/null +++ b/kustomizations/observability-gateway/gateway.yaml @@ -0,0 +1,65 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: observability-gateway +spec: + gatewayClassName: envoy-gateway + listeners: + - name: prometheus + port: 8443 + protocol: HTTPS + hostname: "" + allowedRoutes: + namespaces: + from: All + tls: + mode: Terminate + certificateRefs: + - kind: Secret + name: prometheus-cert + - name: victoria-logs + port: 8443 + protocol: HTTPS + hostname: "" + allowedRoutes: + namespaces: + from: All + tls: + mode: Terminate + certificateRefs: + - kind: Secret + name: victoria-logs-cert +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: ClientTrafficPolicy +metadata: + name: prometheus-mtls +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: observability-gateway + sectionName: prometheus + tls: + clientValidation: + caCertificateRefs: + - kind: "Secret" + group: "" + name: "prometheus-client-ca-cert" +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: ClientTrafficPolicy +metadata: + name: victoria-logs-mtls +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: observability-gateway + sectionName: victoria-logs + tls: + clientValidation: + caCertificateRefs: + - kind: "Secret" + group: "" + name: "victoria-logs-client-ca-cert" diff --git a/kustomizations/observability-gateway/kustomization.yaml b/kustomizations/observability-gateway/kustomization.yaml new file mode 100644 index 0000000..a402d5c --- /dev/null +++ b/kustomizations/observability-gateway/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - gateway.yaml + - gateway-issuer.yaml + - prometheus-certificates.yaml + - victoria-logs-certificates.yaml diff --git a/kustomizations/prometheus/client-certificates.yaml b/kustomizations/observability-gateway/prometheus-certificates.yaml similarity index 75% rename from kustomizations/prometheus/client-certificates.yaml rename to kustomizations/observability-gateway/prometheus-certificates.yaml index c9c9848..9ab1e63 100644 --- a/kustomizations/prometheus/client-certificates.yaml +++ b/kustomizations/observability-gateway/prometheus-certificates.yaml @@ -1,5 +1,17 @@ apiVersion: cert-manager.io/v1 kind: Certificate +metadata: + name: prometheus-gateway-cert +spec: + secretName: prometheus-cert + issuerRef: + name: gateway-selfsigned-issuer + kind: Issuer + dnsNames: + - "" +--- +apiVersion: cert-manager.io/v1 +kind: Certificate metadata: name: prometheus-client-ca spec: diff --git a/kustomizations/observability-gateway/victoria-logs-certificates.yaml b/kustomizations/observability-gateway/victoria-logs-certificates.yaml new file mode 100644 index 0000000..3e4a91d --- /dev/null +++ b/kustomizations/observability-gateway/victoria-logs-certificates.yaml @@ -0,0 +1,47 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: victoria-logs-gateway-cert +spec: + secretName: victoria-logs-cert + issuerRef: + name: gateway-selfsigned-issuer + kind: Issuer + dnsNames: + - "" +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: victoria-logs-client-ca +spec: + isCA: true + commonName: victoria-logs-client-ca + secretName: victoria-logs-client-ca-cert + privateKey: + algorithm: RSA + size: 2048 + issuerRef: + name: gateway-selfsigned-issuer + kind: Issuer +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: victoria-logs-client-issuer +spec: + ca: + secretName: victoria-logs-client-ca-cert +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: victoria-logs-client-cert +spec: + secretName: victoria-logs-client-cert + commonName: victoria-logs-client + usages: + - client auth + issuerRef: + name: victoria-logs-client-issuer + kind: Issuer diff --git a/kustomizations/opentelemetry-collector/kustomization.yaml b/kustomizations/opentelemetry-collector/kustomization.yaml index a1320d5..c09d510 100644 --- a/kustomizations/opentelemetry-collector/kustomization.yaml +++ b/kustomizations/opentelemetry-collector/kustomization.yaml @@ -2,4 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - collector.yaml + - log-collector.yaml - servicemonitor.yaml diff --git a/kustomizations/opentelemetry-collector/log-collector.yaml b/kustomizations/opentelemetry-collector/log-collector.yaml new file mode 100644 index 0000000..413007f --- /dev/null +++ b/kustomizations/opentelemetry-collector/log-collector.yaml @@ -0,0 +1,111 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: open-telemetry-log-collector +--- +apiVersion: opentelemetry.io/v1beta1 +kind: OpenTelemetryCollector +metadata: + name: logs +spec: + mode: daemonset + serviceAccount: open-telemetry-log-collector + securityContext: + runAsUser: 0 + config: + receivers: + filelog: + include: + - /var/log/pods/*/*/*.log + start_at: beginning + include_file_path: true + include_file_name: false + operators: + # Route to the correct parser based on container runtime format + - type: router + id: get-format + routes: + - output: parser-docker + expr: 'body matches "^\\{"' + default: parser-containerd + + # Docker JSON format (e.g. Docker Desktop, older clusters) + - type: json_parser + id: parser-docker + output: move-log-to-body + timestamp: + parse_from: attributes.time + layout: '%Y-%m-%dT%H:%M:%S.%LZ' + + # Containerd / CRI-O space-delimited format (most modern clusters) + - type: regex_parser + id: parser-containerd + regex: '^(?P