Skip to content

feat: elasticsearch exporter#193

Open
csatib02 wants to merge 7 commits intofeat/output-batchingfrom
feat/elasticsearch-exporter
Open

feat: elasticsearch exporter#193
csatib02 wants to merge 7 commits intofeat/output-batchingfrom
feat/elasticsearch-exporter

Conversation

@csatib02
Copy link
Member

@csatib02 csatib02 commented Mar 4, 2026

Adds support for the elasticsearch exporter.

The current handling of secrets, such as credentials from basicAuth or bearerAuth extension were stored as plaintext in config. I've adjusted it in a non breaking way.

  • The secrets can be mounted from a secret just as before.
  • The operator manages a secret/collector, that contains the same values, but under different keys.
  • The keys are mounted as env-vars in the collector, essentially hiding them.

Manually tested with the commited demo.

curl -sk -u elastic:qG6N8rDL80QYLuZp -X POST "https://localhost:9200/logs-generic-default/_search?pretty" \
  -H 'Content-Type: application/json' \
  -d '{"size": 5, "sort": [{"@timestamp": {"order": "desc"}}]}'
{
  "took" : 965,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 2007,
      "relation" : "eq"
    },
    "max_score" : null,
    "hits" : [
      {
        "_index" : ".ds-logs-generic-default-2026.03.04-000001",
        "_id" : "9TFluZwBSf2q5Fw9J94V",
        "_score" : null,
        "_source" : {
          "@timestamp" : "2026-03-04T15:08:59.889076097Z",
          "agent" : {
            "name" : "otlp"
          },
          "data_stream" : {
            "dataset" : "generic",
            "namespace" : "default",
            "type" : "logs"
          },
          "exporter" : "elasticsearch/tenant-es-demo_elasticsearch",
          "k8s" : {
            "container" : {
              "restart_count" : "0"
            },
            "pod" : {
              "labels" : {
                "app" : {
                  "kubernetes" : {
                    "io/instance" : "log-generator",
                    "io/name" : "log-generator"
                  }
                },
                "pod-template-hash" : "988b67d7c"
              },
              "start_time" : "2026-03-04T14:07:27Z"
            }
          },
          "kubernetes" : {
            "container" : {
              "name" : "log-generator"
            },
            "deployment" : {
              "name" : "log-generator"
            },
            "namespace" : "tenant-es-demo",
            "node" : {
              "name" : "kind-control-plane"
            },
            "pod" : {
              "name" : "log-generator-988b67d7c-gdx4b",
              "uid" : "8819071a-0246-4f1a-bb38-f779b3faf56e"
            }
          },
          "log" : {
            "file" : {
              "path" : "/var/log/pods/tenant-es-demo_log-generator-988b67d7c-gdx4b_8819071a-0246-4f1a-bb38-f779b3faf56e/log-generator/0.log"
            },
            "iostream" : "stdout"
          },
          "logtag" : "F",
          "message" : "112.222.204.97 - - [04/Mar/2026:15:08:59 +0000] \"GET /blog HTTP/1.1\" 404 15589 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:52.0) Gecko/20100101 Firefox/52.0\" \"-\"",
          "subscription" : "all-logs",
          "tenant" : "es-demo",
          "time" : "2026-03-04T15:08:59.889076097Z"
        },
        "sort" : [
          1772636939889
        ]
      },
      {
        "_index" : ".ds-logs-generic-default-2026.03.04-000001",
        "_id" : "9DFluZwBSf2q5Fw9J94V",
        "_score" : null,
        "_source" : {
          "@timestamp" : "2026-03-04T15:08:58.888422097Z",
          "agent" : {
            "name" : "otlp"
          },
          "data_stream" : {
            "dataset" : "generic",
            "namespace" : "default",
            "type" : "logs"
          },
          "exporter" : "elasticsearch/tenant-es-demo_elasticsearch",
          "k8s" : {
            "container" : {
              "restart_count" : "0"
            },
            "pod" : {
              "labels" : {
                "app" : {
                  "kubernetes" : {
                    "io/instance" : "log-generator",
                    "io/name" : "log-generator"
                  }
                },
                "pod-template-hash" : "988b67d7c"
              },
              "start_time" : "2026-03-04T14:07:27Z"
            }
          },
          "kubernetes" : {
            "container" : {
              "name" : "log-generator"
            },
            "deployment" : {
              "name" : "log-generator"
            },
            "namespace" : "tenant-es-demo",
            "node" : {
              "name" : "kind-control-plane"
            },
            "pod" : {
              "name" : "log-generator-988b67d7c-gdx4b",
              "uid" : "8819071a-0246-4f1a-bb38-f779b3faf56e"
            }
          },
          "log" : {
            "file" : {
              "path" : "/var/log/pods/tenant-es-demo_log-generator-988b67d7c-gdx4b_8819071a-0246-4f1a-bb38-f779b3faf56e/log-generator/0.log"
            },
            "iostream" : "stdout"
          },
          "logtag" : "F",
          "message" : "137.16.94.135 - - [04/Mar/2026:15:08:58 +0000] \"PUT /index.html HTTP/1.1\" 404 19314 \"-\" \"Mozilla/5.0 (Linux; Android 8.0.0; Pixel XL Build/OPR6.170623.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.107 Mobile Safari/537.36\" \"-\"",
          "subscription" : "all-logs",
          "tenant" : "es-demo",
          "time" : "2026-03-04T15:08:58.888422097Z"
        },
        "sort" : [
          1772636938888
        ]
      },
      {
        "_index" : ".ds-logs-generic-default-2026.03.04-000001",
        "_id" : "8zFluZwBSf2q5Fw9J94V",
        "_score" : null,
        "_source" : {
          "@timestamp" : "2026-03-04T15:08:57.886356304Z",
          "agent" : {
            "name" : "otlp"
          },
          "data_stream" : {
            "dataset" : "generic",
            "namespace" : "default",
            "type" : "logs"
          },
          "exporter" : "elasticsearch/tenant-es-demo_elasticsearch",
          "k8s" : {
            "container" : {
              "restart_count" : "0"
            },
            "pod" : {
              "labels" : {
                "app" : {
                  "kubernetes" : {
                    "io/instance" : "log-generator",
                    "io/name" : "log-generator"
                  }
                },
                "pod-template-hash" : "988b67d7c"
              },
              "start_time" : "2026-03-04T14:07:27Z"
            }
          },
          "kubernetes" : {
            "container" : {
              "name" : "log-generator"
            },
            "deployment" : {
              "name" : "log-generator"
            },
            "namespace" : "tenant-es-demo",
            "node" : {
              "name" : "kind-control-plane"
            },
            "pod" : {
              "name" : "log-generator-988b67d7c-gdx4b",
              "uid" : "8819071a-0246-4f1a-bb38-f779b3faf56e"
            }
          },
          "log" : {
            "file" : {
              "path" : "/var/log/pods/tenant-es-demo_log-generator-988b67d7c-gdx4b_8819071a-0246-4f1a-bb38-f779b3faf56e/log-generator/0.log"
            },
            "iostream" : "stdout"
          },
          "logtag" : "F",
          "message" : "15.45.109.46 - - [04/Mar/2026:15:08:57 +0000] \"GET / HTTP/1.1\" 403 5749 \"-\" \"Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0 Iceweasel/38.2.1\" \"-\"",
          "subscription" : "all-logs",
          "tenant" : "es-demo",
          "time" : "2026-03-04T15:08:57.886356304Z"
        },
        "sort" : [
          1772636937886
        ]
      },
      {
        "_index" : ".ds-logs-generic-default-2026.03.04-000001",
        "_id" : "8jFluZwBSf2q5Fw9J94V",
        "_score" : null,
        "_source" : {
          "@timestamp" : "2026-03-04T15:08:56.886040804Z",
          "agent" : {
            "name" : "otlp"
          },
          "data_stream" : {
            "dataset" : "generic",
            "namespace" : "default",
            "type" : "logs"
          },
          "exporter" : "elasticsearch/tenant-es-demo_elasticsearch",
          "k8s" : {
            "container" : {
              "restart_count" : "0"
            },
            "pod" : {
              "labels" : {
                "app" : {
                  "kubernetes" : {
                    "io/instance" : "log-generator",
                    "io/name" : "log-generator"
                  }
                },
                "pod-template-hash" : "988b67d7c"
              },
              "start_time" : "2026-03-04T14:07:27Z"
            }
          },
          "kubernetes" : {
            "container" : {
              "name" : "log-generator"
            },
            "deployment" : {
              "name" : "log-generator"
            },
            "namespace" : "tenant-es-demo",
            "node" : {
              "name" : "kind-control-plane"
            },
            "pod" : {
              "name" : "log-generator-988b67d7c-gdx4b",
              "uid" : "8819071a-0246-4f1a-bb38-f779b3faf56e"
            }
          },
          "log" : {
            "file" : {
              "path" : "/var/log/pods/tenant-es-demo_log-generator-988b67d7c-gdx4b_8819071a-0246-4f1a-bb38-f779b3faf56e/log-generator/0.log"
            },
            "iostream" : "stdout"
          },
          "logtag" : "F",
          "message" : "4.31.78.98 - - [04/Mar/2026:15:08:56 +0000] \"PUT /products HTTP/1.1\" 403 6241 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36\" \"-\"",
          "subscription" : "all-logs",
          "tenant" : "es-demo",
          "time" : "2026-03-04T15:08:56.886040804Z"
        },
        "sort" : [
          1772636936886
        ]
      },
      {
        "_index" : ".ds-logs-generic-default-2026.03.04-000001",
        "_id" : "8TFluZwBSf2q5Fw9J94V",
        "_score" : null,
        "_source" : {
          "@timestamp" : "2026-03-04T15:08:55.884651970Z",
          "agent" : {
            "name" : "otlp"
          },
          "data_stream" : {
            "dataset" : "generic",
            "namespace" : "default",
            "type" : "logs"
          },
          "exporter" : "elasticsearch/tenant-es-demo_elasticsearch",
          "k8s" : {
            "container" : {
              "restart_count" : "0"
            },
            "pod" : {
              "labels" : {
                "app" : {
                  "kubernetes" : {
                    "io/instance" : "log-generator",
                    "io/name" : "log-generator"
                  }
                },
                "pod-template-hash" : "988b67d7c"
              },
              "start_time" : "2026-03-04T14:07:27Z"
            }
          },
          "kubernetes" : {
            "container" : {
              "name" : "log-generator"
            },
            "deployment" : {
              "name" : "log-generator"
            },
            "namespace" : "tenant-es-demo",
            "node" : {
              "name" : "kind-control-plane"
            },
            "pod" : {
              "name" : "log-generator-988b67d7c-gdx4b",
              "uid" : "8819071a-0246-4f1a-bb38-f779b3faf56e"
            }
          },
          "log" : {
            "file" : {
              "path" : "/var/log/pods/tenant-es-demo_log-generator-988b67d7c-gdx4b_8819071a-0246-4f1a-bb38-f779b3faf56e/log-generator/0.log"
            },
            "iostream" : "stdout"
          },
          "logtag" : "F",
          "message" : "195.78.147.249 - - [04/Mar/2026:15:08:55 +0000] \"POST / HTTP/1.1\" 404 11170 \"-\" \"Mozilla/5.0 (Linux; Android 8.0.0; Pixel XL Build/OPR6.170623.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.107 Mobile Safari/537.36\" \"-\"",
          "subscription" : "all-logs",
          "tenant" : "es-demo",
          "time" : "2026-03-04T15:08:55.88465197Z"
        },
        "sort" : [
          1772636935884
        ]
      }
    ]
  }
}

csatib02 added 7 commits March 3, 2026 17:27
Signed-off-by: Bence Csati <bence.csati@axoflow.com>
Signed-off-by: Bence Csati <bence.csati@axoflow.com>
Signed-off-by: Bence Csati <bence.csati@axoflow.com>
Signed-off-by: Bence Csati <bence.csati@axoflow.com>
Signed-off-by: Bence Csati <bence.csati@axoflow.com>
Signed-off-by: Bence Csati <bence.csati@axoflow.com>
Signed-off-by: Bence Csati <bence.csati@axoflow.com>
@csatib02 csatib02 requested review from OverOrion and pepov March 4, 2026 15:06
@csatib02 csatib02 self-assigned this Mar 4, 2026
@csatib02 csatib02 added the enhancement New feature or request label Mar 4, 2026
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

enhancement New feature or request

Projects

None yet

Development

Successfully merging this pull request may close these issues.

1 participant