From 183b2e5c46c721df71c867855c4c793b541e71eb Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Thu, 2 Apr 2026 16:16:54 +0200 Subject: [PATCH 1/8] tests: disable 02985_dialects_with_distributed_tables It requires PRQL that is disabled in FIPS. --- .../0_stateless/02985_dialects_with_distributed_tables.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql index e9e6934f13cb..28d9bb2cf326 100644 --- a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql +++ b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest, distributed +-- Tags: disabled, no-fasttest, distributed SET allow_experimental_prql_dialect = 1; SET allow_experimental_kusto_dialect = 1; From eb89c3e299a6a5a91b557f3679bb722ad09ae604 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Fri, 3 Apr 2026 09:27:34 +0200 Subject: [PATCH 2/8] tests: skip test_storage_delta DeltaLake is not supported in FIPS. --- tests/integration/test_storage_delta/test.py | 2 ++ tests/integration/test_storage_delta/test_imds.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/integration/test_storage_delta/test.py b/tests/integration/test_storage_delta/test.py index f5a23ab807eb..a5dc74d8447f 100644 --- a/tests/integration/test_storage_delta/test.py +++ b/tests/integration/test_storage_delta/test.py @@ -47,6 +47,8 @@ from helpers.config_cluster import minio_access_key from helpers.config_cluster import minio_secret_key +pytestmark = pytest.mark.skip(reason="DeltaLake not supported in FIPS") + SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_storage_delta/test_imds.py b/tests/integration/test_storage_delta/test_imds.py index 257de0d6778d..feb4e79cd999 100644 --- a/tests/integration/test_storage_delta/test_imds.py +++ b/tests/integration/test_storage_delta/test_imds.py @@ -34,6 +34,8 @@ prepare_s3_bucket, ) +pytestmark = pytest.mark.skip(reason="DeltaLake not supported in FIPS") + SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) METADATA_SERVER_HOSTNAME = "resolver" METADATA_SERVER_PORT = 8080 From 5dd7632e25d9892c8354d3380fbe38a1cb254924 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Fri, 3 Apr 2026 09:41:44 +0200 Subject: [PATCH 3/8] tests: fix test_replicated_merge_tree_encryption_codec We replaced aes_128_gcm_siv with aes_128_gcm, but it was not reflected in the test configs. Let's fix it. --- .../configs/key_a.xml | 4 ++-- .../configs/key_a_and_b_current_a.xml | 4 ++-- .../configs/key_a_and_b_current_b.xml | 4 ++-- .../configs/key_a_and_nonce_x.xml | 4 ++-- .../configs/key_a_and_nonce_y.xml | 4 ++-- .../configs/key_b.xml | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a.xml b/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a.xml index a31978e70155..586af598adaa 100644 --- a/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a.xml +++ b/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a.xml @@ -1,7 +1,7 @@ - + aaaaaaaaaaaaaaaa - + diff --git a/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a_and_b_current_a.xml b/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a_and_b_current_a.xml index 01ca9123ccbc..8dd190649332 100644 --- a/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a_and_b_current_a.xml +++ b/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a_and_b_current_a.xml @@ -1,10 +1,10 @@ - + aaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbb 0 - + diff --git a/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a_and_b_current_b.xml b/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a_and_b_current_b.xml index 98cf6ced0c75..ff161c3776e1 100644 --- a/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a_and_b_current_b.xml +++ b/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a_and_b_current_b.xml @@ -1,10 +1,10 @@ - + aaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbb 1 - + diff --git a/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a_and_nonce_x.xml b/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a_and_nonce_x.xml index 40c5adab19b9..715c9a77e98c 100644 --- a/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a_and_nonce_x.xml +++ b/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a_and_nonce_x.xml @@ -1,8 +1,8 @@ - + aaaaaaaaaaaaaaaa xxxxxxxxxxxx - + diff --git a/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a_and_nonce_y.xml b/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a_and_nonce_y.xml index eadfb6e67338..713057c4e223 100644 --- a/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a_and_nonce_y.xml +++ b/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_a_and_nonce_y.xml @@ -1,8 +1,8 @@ - + aaaaaaaaaaaaaaaa yyyyyyyyyyyy - + diff --git a/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_b.xml b/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_b.xml index e336324f648c..0060d720e242 100644 --- a/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_b.xml +++ b/tests/integration/test_replicated_merge_tree_encryption_codec/configs/key_b.xml @@ -1,7 +1,7 @@ - + bbbbbbbbbbbbbbbb - + From 19b6cb3ed650c9649aa6b0f20e7c9ca2a03eeba8 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Sat, 4 Apr 2026 10:41:45 +0200 Subject: [PATCH 4/8] lsan: suppress AWS-LC FIPS service indicator false positive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AWS-LC FIPS 2.0.0 allocates a per-thread `fips_service_indicator_state` struct (48 bytes + 24-byte pointer array) on the first FIPS-approved crypto operation via `service_indicator_get()` → `CRYPTO_set_thread_local()` → `OPENSSL_malloc()`. This state tracks whether each cryptographic operation used a FIPS-approved algorithm, as required by FIPS 140-3 compliance. The memory is registered with a pthread TLS destructor (`OPENSSL_free`) that fires when the thread exits. However, in ClickHouse the crypto operations (e.g. SHA-256 for S3 request signing) run on GlobalThreadPool worker threads. These worker threads outlive the application-level thread pools (like `threadpool_writer`) because `ThreadFromGlobalPoolImpl` submits jobs to the GlobalThreadPool rather than owning threads directly. The GlobalThreadPool worker threads are only joined during static destruction of `GlobalThreadPool::the_instance`, which races with LeakSanitizer's atexit check. This is a false positive: the memory IS freed when the worker threads eventually exit, but LSAN runs its scan before GlobalThreadPool static destruction completes. This issue is FIPS-specific: the `FIPS_service_indicator_update_state` code path only exists in AWS-LC FIPS builds. Non-FIPS builds (OpenSSL 3.2.1) do not have service indicator tracking. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/config/lsan_suppressions.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/config/lsan_suppressions.txt b/tests/config/lsan_suppressions.txt index 39eb40560d7e..0b9274f980a7 100644 --- a/tests/config/lsan_suppressions.txt +++ b/tests/config/lsan_suppressions.txt @@ -1,2 +1,6 @@ # See https://bugs.llvm.org/show_bug.cgi?id=47418 # leak:getActualTableStructure + +# AWS-LC FIPS 2.0.0: suppress per-thread FIPS service indicator state leak. +# https://github.com/aws/aws-lc/blob/AWS-LC-FIPS-2.0.0/crypto/fipsmodule/service_indicator/service_indicator.c#L57 +leak:service_indicator_get From bf4af17d8f73007424c64e803c7b6aa379cab73d Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Sat, 4 Apr 2026 11:14:35 +0200 Subject: [PATCH 5/8] tests: add LSAN suppressions for AWS-LC FIPS integration tests AWS-LC FIPS 2.0.0 allocates a per-thread `fips_service_indicator_state` (48 + 24 bytes) on the first FIPS-approved crypto operation via `service_indicator_get()`. This state is freed by a pthread TLS destructor when the thread exits, but in ClickHouse the crypto operations (e.g. SHA-256 for S3 request signing) run on GlobalThreadPool worker threads whose lifetime extends beyond LeakSanitizer's check. This causes LSAN to report a false positive in integration tests that invoke `clickhouse disks` or other CLI tools performing S3 operations. The fix: - Add `tests/integration/helpers/lsan_suppressions.txt` with a suppression for `service_indicator_get` (placed in helpers/ because only `tests/integration/` is mounted into the test runner container) - Configure `cluster.py` to set `LSAN_OPTIONS` with the suppressions file path and copy it into each instance's config directory This issue is FIPS-specific: the `FIPS_service_indicator_update_state` code path only exists in AWS-LC FIPS builds. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/config/lsan_suppressions.txt | 4 ---- tests/integration/helpers/cluster.py | 14 ++++++++++++++ tests/integration/helpers/lsan_suppressions.txt | 7 +++++++ 3 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 tests/integration/helpers/lsan_suppressions.txt diff --git a/tests/config/lsan_suppressions.txt b/tests/config/lsan_suppressions.txt index 0b9274f980a7..39eb40560d7e 100644 --- a/tests/config/lsan_suppressions.txt +++ b/tests/config/lsan_suppressions.txt @@ -1,6 +1,2 @@ # See https://bugs.llvm.org/show_bug.cgi?id=47418 # leak:getActualTableStructure - -# AWS-LC FIPS 2.0.0: suppress per-thread FIPS service indicator state leak. -# https://github.com/aws/aws-lc/blob/AWS-LC-FIPS-2.0.0/crypto/fipsmodule/service_indicator/service_indicator.c#L57 -leak:service_indicator_get diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 337403af0646..cb7197ce61be 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -469,6 +469,16 @@ def __init__( # [1]: https://github.com/ClickHouse/ClickHouse/issues/43426#issuecomment-1368512678 self.env_variables["ASAN_OPTIONS"] = "use_sigaltstack=0" self.env_variables["TSAN_OPTIONS"] = "use_sigaltstack=0" + lsan_suppressions_file = p.abspath( + p.join(HELPERS_DIR, "lsan_suppressions.txt") + ) + if p.exists(lsan_suppressions_file): + self.lsan_suppressions_file = lsan_suppressions_file + self.env_variables["LSAN_OPTIONS"] = ( + "suppressions=/etc/clickhouse-server/lsan_suppressions.txt" + ) + else: + self.lsan_suppressions_file = None self.env_variables["CLICKHOUSE_WATCHDOG_ENABLE"] = "0" self.env_variables["CLICKHOUSE_NATS_TLS_SECURE"] = "0" self.up_called = False @@ -4737,6 +4747,10 @@ def write_embedded_config(name, dest_dir, fix_log_level=False): self.coredns_config_dir, p.abspath(p.join(self.path, "coredns_config")) ) + # Copy LSAN suppressions if available (mounted at /etc/clickhouse-server/) + if self.cluster.lsan_suppressions_file: + shutil.copy(self.cluster.lsan_suppressions_file, instance_config_dir) + # Copy config.d configs logging.debug( f"Copy custom test config files {self.custom_main_config_paths} to {self.config_d_dir}" diff --git a/tests/integration/helpers/lsan_suppressions.txt b/tests/integration/helpers/lsan_suppressions.txt new file mode 100644 index 000000000000..04c37e7efec8 --- /dev/null +++ b/tests/integration/helpers/lsan_suppressions.txt @@ -0,0 +1,7 @@ +# AWS-LC FIPS 2.0.0: suppress per-thread FIPS service indicator state leak. +# The service indicator tracks whether each crypto operation used a FIPS-approved +# algorithm. It is allocated via CRYPTO_set_thread_local on first use and freed +# by a pthread TLS destructor when the thread exits. In ClickHouse, the crypto +# operations run on GlobalThreadPool worker threads that outlive LSAN's check. +# https://github.com/aws/aws-lc/blob/AWS-LC-FIPS-2.0.0/crypto/fipsmodule/service_indicator/service_indicator.c#L57 +leak:service_indicator_get From 7af86094a46ffc56047b8cc3e8a8d3706e1be956 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Sat, 4 Apr 2026 13:52:31 +0200 Subject: [PATCH 6/8] tests: enable LSAN suppressions for stateless tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit (bf4af17d8f7) added LSAN suppressions for integration tests via cluster.py, but stateless tests inherit LSAN_OPTIONS from the Docker image's ENV directive which did not include the suppressions file path. The base Dockerfile already wrote the correct LSAN_OPTIONS (with suppressions path) to /etc/environment (line 38) for services, but the ENV on line 45 — used by non-login shells including the test runner — omitted the suppressions= directive. This caused clickhouse-disks and other CLI tools to abort when LSAN detected the AWS-LC FIPS service_indicator_get allocation. Fix both: - docker/test/base/Dockerfile: add suppressions path to ENV LSAN_OPTIONS, matching the /etc/environment entry - tests/config/lsan_suppressions.txt: add service_indicator_get suppression (this file is installed at /usr/share/clickhouse-test/config/lsan_suppressions.txt in the test Docker image) Co-Authored-By: Claude Opus 4.6 (1M context) --- docker/test/base/Dockerfile | 2 +- tests/config/lsan_suppressions.txt | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index 26cd016a5abc..a8b47a63e251 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -42,7 +42,7 @@ RUN echo "ASAN_OPTIONS='halt_on_error=1 abort_on_error=1'" >> /etc/environment ENV TSAN_OPTIONS='halt_on_error=1 abort_on_error=1 history_size=7 memory_limit_mb=46080 second_deadlock_stack=1 max_allocation_size_mb=32768' ENV UBSAN_OPTIONS='print_stacktrace=1 max_allocation_size_mb=32768' ENV MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1 max_allocation_size_mb=32768' -ENV LSAN_OPTIONS='max_allocation_size_mb=32768' +ENV LSAN_OPTIONS='suppressions=/usr/share/clickhouse-test/config/lsan_suppressions.txt max_allocation_size_mb=32768' ENV ASAN_OPTIONS='halt_on_error=1 abort_on_error=1' # for external_symbolizer_path, and also ensure that llvm-symbolizer really diff --git a/tests/config/lsan_suppressions.txt b/tests/config/lsan_suppressions.txt index 39eb40560d7e..0b9274f980a7 100644 --- a/tests/config/lsan_suppressions.txt +++ b/tests/config/lsan_suppressions.txt @@ -1,2 +1,6 @@ # See https://bugs.llvm.org/show_bug.cgi?id=47418 # leak:getActualTableStructure + +# AWS-LC FIPS 2.0.0: suppress per-thread FIPS service indicator state leak. +# https://github.com/aws/aws-lc/blob/AWS-LC-FIPS-2.0.0/crypto/fipsmodule/service_indicator/service_indicator.c#L57 +leak:service_indicator_get From c3b003db8b8218f1c8e76e4ca557437a3bbbc79f Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Sat, 4 Apr 2026 14:25:13 +0200 Subject: [PATCH 7/8] docker: refresh expired kitware GPG key in test-base image The kitware apt key baked into the altinityinfra/test-util base image has expired, causing `apt-get update` to fail with: GPG error: https://apt.kitware.com/ubuntu jammy InRelease: NO_PUBKEY 65ADECD7A7039392 Re-fetch the key before the first apt-get update, same approach as 53bcc4f6e809 and 4bc620d0cbde for the binary-builder image. Co-Authored-By: Claude Opus 4.6 (1M context) --- docker/test/base/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index a8b47a63e251..d015f4ce82ad 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -3,6 +3,10 @@ ARG FROM_TAG=latest FROM altinityinfra/test-util:$FROM_TAG +# Refresh the Kitware GPG key inherited from the base image (may have expired) +RUN rm -f /etc/apt/trusted.gpg.d/kitware.gpg \ + && curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc | gpg --batch --dearmor -o /etc/apt/trusted.gpg.d/kitware.gpg + RUN apt-get update \ && apt-get install \ lcov \ From 50ffdb249e0443a58b03022b5755ad83c62ace29 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Sat, 4 Apr 2026 14:57:57 +0200 Subject: [PATCH 8/8] tests: set LSAN suppressions in clickhouse-test runner Set LSAN_OPTIONS with the suppressions file path in the stateless test runner (tests/clickhouse-test) instead of modifying the Docker image. This ensures clickhouse-disks and other CLI tools invoked from stateless tests pick up the AWS-LC FIPS service_indicator_get suppression. Also reverts the docker/test/base/Dockerfile changes from the previous two commits (LSAN_OPTIONS ENV and kitware GPG key refresh) to avoid triggering Docker image rebuilds. Co-Authored-By: Claude Opus 4.6 (1M context) --- docker/test/base/Dockerfile | 6 +----- tests/clickhouse-test | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index d015f4ce82ad..26cd016a5abc 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -3,10 +3,6 @@ ARG FROM_TAG=latest FROM altinityinfra/test-util:$FROM_TAG -# Refresh the Kitware GPG key inherited from the base image (may have expired) -RUN rm -f /etc/apt/trusted.gpg.d/kitware.gpg \ - && curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc | gpg --batch --dearmor -o /etc/apt/trusted.gpg.d/kitware.gpg - RUN apt-get update \ && apt-get install \ lcov \ @@ -46,7 +42,7 @@ RUN echo "ASAN_OPTIONS='halt_on_error=1 abort_on_error=1'" >> /etc/environment ENV TSAN_OPTIONS='halt_on_error=1 abort_on_error=1 history_size=7 memory_limit_mb=46080 second_deadlock_stack=1 max_allocation_size_mb=32768' ENV UBSAN_OPTIONS='print_stacktrace=1 max_allocation_size_mb=32768' ENV MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1 max_allocation_size_mb=32768' -ENV LSAN_OPTIONS='suppressions=/usr/share/clickhouse-test/config/lsan_suppressions.txt max_allocation_size_mb=32768' +ENV LSAN_OPTIONS='max_allocation_size_mb=32768' ENV ASAN_OPTIONS='halt_on_error=1 abort_on_error=1' # for external_symbolizer_path, and also ensure that llvm-symbolizer really diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 9c4552f549e9..4594df14a9fd 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -1830,6 +1830,7 @@ class TestCase: "ASAN_OPTIONS", "MSAN_OPTIONS", "UBSAN_OPTIONS", + "LSAN_OPTIONS", ]: current_options = os.environ.get(env_name, None) if current_options is None: @@ -1837,6 +1838,20 @@ class TestCase: elif "log_path=" not in current_options: os.environ[env_name] += f":log_path={args.client_log}" + # Ensure LSAN picks up the suppressions file if available + lsan_suppressions = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "config", + "lsan_suppressions.txt", + ) + if os.path.exists(lsan_suppressions): + current = os.environ.get("LSAN_OPTIONS", "") + if "suppressions=" not in current: + sep = ":" if current else "" + os.environ["LSAN_OPTIONS"] = ( + current + sep + f"suppressions={lsan_suppressions}" + ) + os.environ["CLICKHOUSE_CLIENT_OPT"] = ( os.environ["CLICKHOUSE_CLIENT_OPT"] if "CLICKHOUSE_CLIENT_OPT" in os.environ