From df22ed5929c309256b4bf023b71d3efdc86bf6e5 Mon Sep 17 00:00:00 2001 From: Joost Boonzajer Flaes Date: Sun, 22 Mar 2026 14:04:07 +0200 Subject: [PATCH 01/12] feat: add show_sample_rows tag to override PII-based sample hiding Adds a new tag mechanism that is the inverse of the existing PII tag behavior. When enable_samples_on_show_sample_rows_tags is true, models or columns tagged with show_sample_rows will have their samples shown even when PII tags would otherwise suppress them. Co-Authored-By: Claude Sonnet 4.6 --- macros/edr/materializations/test/test.sql | 8 ++++--- .../system/system_utils/get_config_var.sql | 2 ++ .../get_pii_columns_from_parent_model.sql | 12 ++++++++++ .../is_show_sample_rows_table.sql | 24 +++++++++++++++++++ 4 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 macros/edr/system/system_utils/is_show_sample_rows_table.sql diff --git a/macros/edr/materializations/test/test.sql b/macros/edr/materializations/test/test.sql index b9a167a72..e860dc5be 100644 --- a/macros/edr/materializations/test/test.sql +++ b/macros/edr/materializations/test/test.sql @@ -76,9 +76,11 @@ {% endif %} {% if disable_test_samples %} {% set sample_limit = 0 %} - {% elif elementary.is_pii_table(flattened_test) %} {% set sample_limit = 0 %} - {% elif elementary.should_disable_sampling_for_pii(flattened_test) %} - {% set sample_limit = 0 %} + {% elif not elementary.is_show_sample_rows_table(flattened_test) %} + {% if elementary.is_pii_table(flattened_test) %} {% set sample_limit = 0 %} + {% elif elementary.should_disable_sampling_for_pii(flattened_test) %} + {% set sample_limit = 0 %} + {% endif %} {% endif %} {% set result_rows = elementary.query_test_result_rows( diff --git a/macros/edr/system/system_utils/get_config_var.sql b/macros/edr/system/system_utils/get_config_var.sql index fc3e1efb1..391ce25d4 100644 --- a/macros/edr/system/system_utils/get_config_var.sql +++ b/macros/edr/system/system_utils/get_config_var.sql @@ -143,6 +143,8 @@ "anomaly_exclude_metrics": none, "disable_samples_on_pii_tags": false, "pii_tags": ["pii"], + "enable_samples_on_show_sample_rows_tags": false, + "show_sample_rows_tags": ["show_sample_rows"], "bigquery_disable_partitioning": false, "bigquery_disable_clustering": false, "upload_only_current_project_artifacts": false, diff --git a/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql b/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql index bae1aea2a..02a779be7 100644 --- a/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql +++ b/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql @@ -38,9 +38,21 @@ {% set column_nodes = parent_model.get("columns") %} {% if not column_nodes %} {% do return(pii_columns) %} {% endif %} + {% set enable_show_tags = elementary.get_config_var("enable_samples_on_show_sample_rows_tags") %} + {% set raw_show_tags = elementary.get_config_var("show_sample_rows_tags") %} + {% set show_tags = ( + (raw_show_tags if raw_show_tags is iterable else [raw_show_tags]) + | map("lower") | list + ) %} + {% for column_node in column_nodes.values() %} {% set all_column_tags_lower = elementary.get_column_tags(column_node) %} + {# Skip columns explicitly tagged to show sample rows (if feature is enabled) #} + {% if enable_show_tags and elementary.lists_intersection(all_column_tags_lower, show_tags) | length > 0 %} + {% continue %} + {% endif %} + {% for pii_tag in pii_tags %} {% if pii_tag in all_column_tags_lower %} {% do pii_columns.append(column_node.get("name")) %} {% break %} diff --git a/macros/edr/system/system_utils/is_show_sample_rows_table.sql b/macros/edr/system/system_utils/is_show_sample_rows_table.sql new file mode 100644 index 000000000..3af6319af --- /dev/null +++ b/macros/edr/system/system_utils/is_show_sample_rows_table.sql @@ -0,0 +1,24 @@ +{% macro is_show_sample_rows_table(flattened_test) %} + {% if not elementary.get_config_var("enable_samples_on_show_sample_rows_tags") %} + {% do return(false) %} + {% endif %} + + {% set raw_show_tags = elementary.get_config_var("show_sample_rows_tags") %} + {% set show_tags = ( + (raw_show_tags if raw_show_tags is iterable else [raw_show_tags]) + | map("lower") + | list + ) %} + + {% set raw_model_tags = elementary.insensitive_get_dict_value( + flattened_test, "model_tags", [] + ) %} + {% set model_tags = ( + (raw_model_tags if raw_model_tags is iterable else [raw_model_tags]) + | map("lower") + | list + ) %} + + {% set intersection = elementary.lists_intersection(model_tags, show_tags) %} + {% do return(intersection | length > 0) %} +{% endmacro %} From dfc2003ffc3fb8fa77632384d47da985e2574b1c Mon Sep 17 00:00:00 2001 From: Joost Boonzajer Flaes Date: Sun, 22 Mar 2026 15:03:10 +0200 Subject: [PATCH 02/12] fix: apply sqlfmt formatting to get_pii_columns_from_parent_model.sql --- .../get_pii_columns_from_parent_model.sql | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql b/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql index 02a779be7..ef57e8518 100644 --- a/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql +++ b/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql @@ -38,18 +38,23 @@ {% set column_nodes = parent_model.get("columns") %} {% if not column_nodes %} {% do return(pii_columns) %} {% endif %} - {% set enable_show_tags = elementary.get_config_var("enable_samples_on_show_sample_rows_tags") %} + {% set enable_show_tags = elementary.get_config_var( + "enable_samples_on_show_sample_rows_tags" + ) %} {% set raw_show_tags = elementary.get_config_var("show_sample_rows_tags") %} {% set show_tags = ( (raw_show_tags if raw_show_tags is iterable else [raw_show_tags]) - | map("lower") | list + | map("lower") + | list ) %} {% for column_node in column_nodes.values() %} {% set all_column_tags_lower = elementary.get_column_tags(column_node) %} {# Skip columns explicitly tagged to show sample rows (if feature is enabled) #} - {% if enable_show_tags and elementary.lists_intersection(all_column_tags_lower, show_tags) | length > 0 %} + {% if enable_show_tags and elementary.lists_intersection( + all_column_tags_lower, show_tags + ) | length > 0 %} {% continue %} {% endif %} From ea16e4734e959c7407719ba6e4790915d2d4f4b3 Mon Sep 17 00:00:00 2001 From: Joost Boonzajer Flaes Date: Sun, 22 Mar 2026 14:06:10 +0200 Subject: [PATCH 03/12] chore: add .claude/ to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index e11a772a8..141381915 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,9 @@ __pycache__/ # vscode .vscode/ + +# Claude Code +.claude/ dbt_internal_packages/ /package-lock.yml From 7dddc7795f8983d2c0d859e9759972152849310c Mon Sep 17 00:00:00 2001 From: Joost Boonzajer Flaes Date: Sun, 22 Mar 2026 15:09:50 +0200 Subject: [PATCH 04/12] fix: use string check for show_sample_rows_tags normalization to avoid char-splitting --- .../system_utils/get_pii_columns_from_parent_model.sql | 8 +++----- .../edr/system/system_utils/is_show_sample_rows_table.sql | 8 +++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql b/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql index ef57e8518..2adbe94ab 100644 --- a/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql +++ b/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql @@ -42,11 +42,9 @@ "enable_samples_on_show_sample_rows_tags" ) %} {% set raw_show_tags = elementary.get_config_var("show_sample_rows_tags") %} - {% set show_tags = ( - (raw_show_tags if raw_show_tags is iterable else [raw_show_tags]) - | map("lower") - | list - ) %} + {% if raw_show_tags is string %} {% set show_tags = [raw_show_tags | lower] %} + {% else %} {% set show_tags = (raw_show_tags or []) | map("lower") | list %} + {% endif %} {% for column_node in column_nodes.values() %} {% set all_column_tags_lower = elementary.get_column_tags(column_node) %} diff --git a/macros/edr/system/system_utils/is_show_sample_rows_table.sql b/macros/edr/system/system_utils/is_show_sample_rows_table.sql index 3af6319af..5eeece300 100644 --- a/macros/edr/system/system_utils/is_show_sample_rows_table.sql +++ b/macros/edr/system/system_utils/is_show_sample_rows_table.sql @@ -4,11 +4,9 @@ {% endif %} {% set raw_show_tags = elementary.get_config_var("show_sample_rows_tags") %} - {% set show_tags = ( - (raw_show_tags if raw_show_tags is iterable else [raw_show_tags]) - | map("lower") - | list - ) %} + {% if raw_show_tags is string %} {% set show_tags = [raw_show_tags | lower] %} + {% else %} {% set show_tags = (raw_show_tags or []) | map("lower") | list %} + {% endif %} {% set raw_model_tags = elementary.insensitive_get_dict_value( flattened_test, "model_tags", [] From 4d7179b6105fad22c48fc47af12eb7a884c7ad4d Mon Sep 17 00:00:00 2001 From: Joost Boonzajer Flaes Date: Tue, 24 Mar 2026 09:06:00 +0200 Subject: [PATCH 05/12] fix: pii tag takes precedence over show_sample_rows tag --- .../get_pii_columns_from_parent_model.sql | 21 +++++++++++++------ .../is_show_sample_rows_table.sql | 9 ++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql b/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql index 2adbe94ab..c94fcbaaa 100644 --- a/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql +++ b/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql @@ -49,12 +49,21 @@ {% for column_node in column_nodes.values() %} {% set all_column_tags_lower = elementary.get_column_tags(column_node) %} - {# Skip columns explicitly tagged to show sample rows (if feature is enabled) #} - {% if enable_show_tags and elementary.lists_intersection( - all_column_tags_lower, show_tags - ) | length > 0 %} - {% continue %} - {% endif %} + {# PII takes precedence: only skip if show_sample_rows is set and pii is not #} + {% set has_show_tag = ( + enable_show_tags + and elementary.lists_intersection( + all_column_tags_lower, show_tags + ) + | length + > 0 + ) %} + {% set has_pii_tag = ( + elementary.lists_intersection(all_column_tags_lower, pii_tags) + | length + > 0 + ) %} + {% if has_show_tag and not has_pii_tag %} {% continue %} {% endif %} {% for pii_tag in pii_tags %} {% if pii_tag in all_column_tags_lower %} diff --git a/macros/edr/system/system_utils/is_show_sample_rows_table.sql b/macros/edr/system/system_utils/is_show_sample_rows_table.sql index 5eeece300..c75834c8b 100644 --- a/macros/edr/system/system_utils/is_show_sample_rows_table.sql +++ b/macros/edr/system/system_utils/is_show_sample_rows_table.sql @@ -17,6 +17,15 @@ | list ) %} + {# PII takes precedence: if model has a PII tag, show_sample_rows is ignored #} + {% set raw_pii_tags = elementary.get_config_var("pii_tags") %} + {% if raw_pii_tags is string %} {% set pii_tags = [raw_pii_tags | lower] %} + {% else %} {% set pii_tags = (raw_pii_tags or []) | map("lower") | list %} + {% endif %} + {% if elementary.lists_intersection(model_tags, pii_tags) | length > 0 %} + {% do return(false) %} + {% endif %} + {% set intersection = elementary.lists_intersection(model_tags, show_tags) %} {% do return(intersection | length > 0) %} {% endmacro %} From 54faaaad06557cc8b3d7fb77fb626fabff54528f Mon Sep 17 00:00:00 2001 From: Joost Boonzajer Flaes Date: Tue, 24 Mar 2026 09:23:54 +0200 Subject: [PATCH 06/12] fix: hide samples by default when enable_samples_on_show_sample_rows_tags is true --- macros/edr/materializations/test/test.sql | 4 +++- .../system_utils/is_show_sample_rows_table.sql | 16 +++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/macros/edr/materializations/test/test.sql b/macros/edr/materializations/test/test.sql index e860dc5be..e151c6d81 100644 --- a/macros/edr/materializations/test/test.sql +++ b/macros/edr/materializations/test/test.sql @@ -77,7 +77,9 @@ {% if disable_test_samples %} {% set sample_limit = 0 %} {% elif not elementary.is_show_sample_rows_table(flattened_test) %} - {% if elementary.is_pii_table(flattened_test) %} {% set sample_limit = 0 %} + {% if elementary.get_config_var("enable_samples_on_show_sample_rows_tags") %} + {% set sample_limit = 0 %} + {% elif elementary.is_pii_table(flattened_test) %} {% set sample_limit = 0 %} {% elif elementary.should_disable_sampling_for_pii(flattened_test) %} {% set sample_limit = 0 %} {% endif %} diff --git a/macros/edr/system/system_utils/is_show_sample_rows_table.sql b/macros/edr/system/system_utils/is_show_sample_rows_table.sql index c75834c8b..7c25d25d3 100644 --- a/macros/edr/system/system_utils/is_show_sample_rows_table.sql +++ b/macros/edr/system/system_utils/is_show_sample_rows_table.sql @@ -17,13 +17,15 @@ | list ) %} - {# PII takes precedence: if model has a PII tag, show_sample_rows is ignored #} - {% set raw_pii_tags = elementary.get_config_var("pii_tags") %} - {% if raw_pii_tags is string %} {% set pii_tags = [raw_pii_tags | lower] %} - {% else %} {% set pii_tags = (raw_pii_tags or []) | map("lower") | list %} - {% endif %} - {% if elementary.lists_intersection(model_tags, pii_tags) | length > 0 %} - {% do return(false) %} + {# PII takes precedence over show_sample_rows, but only when PII hiding is enabled #} + {% if elementary.get_config_var("disable_samples_on_pii_tags") %} + {% set raw_pii_tags = elementary.get_config_var("pii_tags") %} + {% if raw_pii_tags is string %} {% set pii_tags = [raw_pii_tags | lower] %} + {% else %} {% set pii_tags = (raw_pii_tags or []) | map("lower") | list %} + {% endif %} + {% if elementary.lists_intersection(model_tags, pii_tags) | length > 0 %} + {% do return(false) %} + {% endif %} {% endif %} {% set intersection = elementary.lists_intersection(model_tags, show_tags) %} From 57fe5c2e417d7826e4547c3b12c9c01e82c64143 Mon Sep 17 00:00:00 2001 From: Joost Boonzajer Flaes Date: Tue, 24 Mar 2026 09:33:15 +0200 Subject: [PATCH 07/12] feat: expand show_sample_rows to model, column, and test level independently of PII --- macros/edr/materializations/test/test.sql | 13 ++- .../is_show_sample_rows_table.sql | 33 -------- .../system_utils/should_show_sample_rows.sql | 83 +++++++++++++++++++ 3 files changed, 89 insertions(+), 40 deletions(-) delete mode 100644 macros/edr/system/system_utils/is_show_sample_rows_table.sql create mode 100644 macros/edr/system/system_utils/should_show_sample_rows.sql diff --git a/macros/edr/materializations/test/test.sql b/macros/edr/materializations/test/test.sql index e151c6d81..7bdb6ca4b 100644 --- a/macros/edr/materializations/test/test.sql +++ b/macros/edr/materializations/test/test.sql @@ -76,13 +76,12 @@ {% endif %} {% if disable_test_samples %} {% set sample_limit = 0 %} - {% elif not elementary.is_show_sample_rows_table(flattened_test) %} - {% if elementary.get_config_var("enable_samples_on_show_sample_rows_tags") %} - {% set sample_limit = 0 %} - {% elif elementary.is_pii_table(flattened_test) %} {% set sample_limit = 0 %} - {% elif elementary.should_disable_sampling_for_pii(flattened_test) %} - {% set sample_limit = 0 %} - {% endif %} + {% elif elementary.should_show_sample_rows(flattened_test) %} {# show_sample_rows tag overrides default-hide #} + {% elif elementary.get_config_var("enable_samples_on_show_sample_rows_tags") %} + {% set sample_limit = 0 %} + {% elif elementary.is_pii_table(flattened_test) %} {% set sample_limit = 0 %} + {% elif elementary.should_disable_sampling_for_pii(flattened_test) %} + {% set sample_limit = 0 %} {% endif %} {% set result_rows = elementary.query_test_result_rows( diff --git a/macros/edr/system/system_utils/is_show_sample_rows_table.sql b/macros/edr/system/system_utils/is_show_sample_rows_table.sql deleted file mode 100644 index 7c25d25d3..000000000 --- a/macros/edr/system/system_utils/is_show_sample_rows_table.sql +++ /dev/null @@ -1,33 +0,0 @@ -{% macro is_show_sample_rows_table(flattened_test) %} - {% if not elementary.get_config_var("enable_samples_on_show_sample_rows_tags") %} - {% do return(false) %} - {% endif %} - - {% set raw_show_tags = elementary.get_config_var("show_sample_rows_tags") %} - {% if raw_show_tags is string %} {% set show_tags = [raw_show_tags | lower] %} - {% else %} {% set show_tags = (raw_show_tags or []) | map("lower") | list %} - {% endif %} - - {% set raw_model_tags = elementary.insensitive_get_dict_value( - flattened_test, "model_tags", [] - ) %} - {% set model_tags = ( - (raw_model_tags if raw_model_tags is iterable else [raw_model_tags]) - | map("lower") - | list - ) %} - - {# PII takes precedence over show_sample_rows, but only when PII hiding is enabled #} - {% if elementary.get_config_var("disable_samples_on_pii_tags") %} - {% set raw_pii_tags = elementary.get_config_var("pii_tags") %} - {% if raw_pii_tags is string %} {% set pii_tags = [raw_pii_tags | lower] %} - {% else %} {% set pii_tags = (raw_pii_tags or []) | map("lower") | list %} - {% endif %} - {% if elementary.lists_intersection(model_tags, pii_tags) | length > 0 %} - {% do return(false) %} - {% endif %} - {% endif %} - - {% set intersection = elementary.lists_intersection(model_tags, show_tags) %} - {% do return(intersection | length > 0) %} -{% endmacro %} diff --git a/macros/edr/system/system_utils/should_show_sample_rows.sql b/macros/edr/system/system_utils/should_show_sample_rows.sql new file mode 100644 index 000000000..4b44a52e1 --- /dev/null +++ b/macros/edr/system/system_utils/should_show_sample_rows.sql @@ -0,0 +1,83 @@ +{% macro should_show_sample_rows(flattened_test) %} + {% if not elementary.get_config_var("enable_samples_on_show_sample_rows_tags") %} + {% do return(false) %} + {% endif %} + + {% set raw_show_tags = elementary.get_config_var("show_sample_rows_tags") %} + {% if raw_show_tags is string %} {% set show_tags = [raw_show_tags | lower] %} + {% else %} {% set show_tags = (raw_show_tags or []) | map("lower") | list %} + {% endif %} + + {# Resolve PII tags once — only relevant when PII hiding is also enabled #} + {% set check_pii = elementary.get_config_var("disable_samples_on_pii_tags") %} + {% if check_pii %} + {% set raw_pii_tags = elementary.get_config_var("pii_tags") %} + {% if raw_pii_tags is string %} {% set pii_tags = [raw_pii_tags | lower] %} + {% else %} {% set pii_tags = (raw_pii_tags or []) | map("lower") | list %} + {% endif %} + {% else %} {% set pii_tags = [] %} + {% endif %} + + {# Model-level check #} + {% set raw_model_tags = elementary.insensitive_get_dict_value( + flattened_test, "model_tags", [] + ) %} + {% if raw_model_tags is string %} {% set model_tags = [raw_model_tags | lower] %} + {% else %} {% set model_tags = (raw_model_tags or []) | map("lower") | list %} + {% endif %} + {% if elementary.lists_intersection(model_tags, show_tags) | length > 0 %} + {% if check_pii and elementary.lists_intersection( + model_tags, pii_tags + ) | length > 0 %} + {% do return(false) %} + {% endif %} + {% do return(true) %} + {% endif %} + + {# Test-level check #} + {% set raw_test_tags = elementary.insensitive_get_dict_value( + flattened_test, "tags", [] + ) %} + {% if raw_test_tags is string %} {% set test_tags = [raw_test_tags | lower] %} + {% else %} {% set test_tags = (raw_test_tags or []) | map("lower") | list %} + {% endif %} + {% if elementary.lists_intersection(test_tags, show_tags) | length > 0 %} + {% if check_pii and elementary.lists_intersection( + model_tags, pii_tags + ) | length > 0 %} + {% do return(false) %} + {% endif %} + {% do return(true) %} + {% endif %} + + {# Column-level check: only the test's target column #} + {% set test_column_name = elementary.insensitive_get_dict_value( + flattened_test, "test_column_name" + ) %} + {% if test_column_name %} + {% set parent_model_unique_id = elementary.insensitive_get_dict_value( + flattened_test, "parent_model_unique_id" + ) %} + {% set parent_model = elementary.get_node(parent_model_unique_id) %} + {% if parent_model %} + {% set column_nodes = parent_model.get("columns", {}) %} + {% for col_name, col_node in column_nodes.items() %} + {% if col_name | lower == test_column_name | lower %} + {% set col_tags = elementary.get_column_tags(col_node) %} + {% if elementary.lists_intersection( + col_tags, show_tags + ) | length > 0 %} + {% if check_pii and elementary.lists_intersection( + col_tags, pii_tags + ) | length > 0 %} + {% do return(false) %} + {% endif %} + {% do return(true) %} + {% endif %} + {% endif %} + {% endfor %} + {% endif %} + {% endif %} + + {% do return(false) %} +{% endmacro %} From bddae5d8f6afbc64907df790dc182a3f5c41b988 Mon Sep 17 00:00:00 2001 From: Joost Boonzajer Flaes Date: Tue, 24 Mar 2026 09:35:02 +0200 Subject: [PATCH 08/12] feat: add test-level pii tag support for consistent behavior with show_sample_rows --- macros/edr/materializations/test/test.sql | 1 + .../edr/system/system_utils/is_pii_test.sql | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 macros/edr/system/system_utils/is_pii_test.sql diff --git a/macros/edr/materializations/test/test.sql b/macros/edr/materializations/test/test.sql index 7bdb6ca4b..5b6c5dbec 100644 --- a/macros/edr/materializations/test/test.sql +++ b/macros/edr/materializations/test/test.sql @@ -80,6 +80,7 @@ {% elif elementary.get_config_var("enable_samples_on_show_sample_rows_tags") %} {% set sample_limit = 0 %} {% elif elementary.is_pii_table(flattened_test) %} {% set sample_limit = 0 %} + {% elif elementary.is_pii_test(flattened_test) %} {% set sample_limit = 0 %} {% elif elementary.should_disable_sampling_for_pii(flattened_test) %} {% set sample_limit = 0 %} {% endif %} diff --git a/macros/edr/system/system_utils/is_pii_test.sql b/macros/edr/system/system_utils/is_pii_test.sql new file mode 100644 index 000000000..8ca36cf4b --- /dev/null +++ b/macros/edr/system/system_utils/is_pii_test.sql @@ -0,0 +1,19 @@ +{% macro is_pii_test(flattened_test) %} + {% if not elementary.get_config_var("disable_samples_on_pii_tags") %} + {% do return(false) %} + {% endif %} + + {% set raw_pii_tags = elementary.get_config_var("pii_tags") %} + {% if raw_pii_tags is string %} {% set pii_tags = [raw_pii_tags | lower] %} + {% else %} {% set pii_tags = (raw_pii_tags or []) | map("lower") | list %} + {% endif %} + + {% set raw_test_tags = elementary.insensitive_get_dict_value( + flattened_test, "tags", [] + ) %} + {% if raw_test_tags is string %} {% set test_tags = [raw_test_tags | lower] %} + {% else %} {% set test_tags = (raw_test_tags or []) | map("lower") | list %} + {% endif %} + + {% do return(elementary.lists_intersection(test_tags, pii_tags) | length > 0) %} +{% endmacro %} From 77e5034f77a67471a7e5cdb5efbacc534517660b Mon Sep 17 00:00:00 2001 From: Joost Boonzajer Flaes Date: Tue, 24 Mar 2026 11:43:49 +0200 Subject: [PATCH 09/12] docs(code): add explanatory comments to show_sample_rows and PII sampling macros --- macros/edr/materializations/test/test.sql | 13 +++++++- .../get_pii_columns_from_parent_model.sql | 8 ++++- .../edr/system/system_utils/is_pii_test.sql | 5 ++++ .../system_utils/should_show_sample_rows.sql | 30 ++++++++++++++++--- 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/macros/edr/materializations/test/test.sql b/macros/edr/materializations/test/test.sql index 5b6c5dbec..681492c5c 100644 --- a/macros/edr/materializations/test/test.sql +++ b/macros/edr/materializations/test/test.sql @@ -75,8 +75,19 @@ {% set disable_test_samples = flattened_test["meta"]["disable_test_samples"] %} {% endif %} + {# + Sampling control precedence (highest to lowest): + 1. disable_test_samples meta flag — explicit per-test kill switch, always wins. + 2. show_sample_rows tag (model/test/column) — opt-in when + enable_samples_on_show_sample_rows_tags is true. If the tag is present, + skip all further checks and keep the sample_limit. + 3. enable_samples_on_show_sample_rows_tags — hide-by-default mode: if the + feature is on but no show_sample_rows tag was found, disable samples. + 4. PII tag detection (model/test/column) — hide when disable_samples_on_pii_tags + is true and a PII tag is detected at any level. + #} {% if disable_test_samples %} {% set sample_limit = 0 %} - {% elif elementary.should_show_sample_rows(flattened_test) %} {# show_sample_rows tag overrides default-hide #} + {% elif elementary.should_show_sample_rows(flattened_test) %} {% elif elementary.get_config_var("enable_samples_on_show_sample_rows_tags") %} {% set sample_limit = 0 %} {% elif elementary.is_pii_table(flattened_test) %} {% set sample_limit = 0 %} diff --git a/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql b/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql index c94fcbaaa..cb3e444a1 100644 --- a/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql +++ b/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql @@ -38,6 +38,12 @@ {% set column_nodes = parent_model.get("columns") %} {% if not column_nodes %} {% do return(pii_columns) %} {% endif %} + {# + A column tagged show_sample_rows (without pii) should still appear in samples + even when disable_samples_on_pii_tags is active — it is intentionally opted in. + We only skip it from the PII columns list if it does NOT also carry a PII tag, + since PII always takes precedence over show_sample_rows. + #} {% set enable_show_tags = elementary.get_config_var( "enable_samples_on_show_sample_rows_tags" ) %} @@ -49,7 +55,7 @@ {% for column_node in column_nodes.values() %} {% set all_column_tags_lower = elementary.get_column_tags(column_node) %} - {# PII takes precedence: only skip if show_sample_rows is set and pii is not #} + {# Skip column from PII list only if show_sample_rows is set and pii is not #} {% set has_show_tag = ( enable_show_tags and elementary.lists_intersection( diff --git a/macros/edr/system/system_utils/is_pii_test.sql b/macros/edr/system/system_utils/is_pii_test.sql index 8ca36cf4b..a050541eb 100644 --- a/macros/edr/system/system_utils/is_pii_test.sql +++ b/macros/edr/system/system_utils/is_pii_test.sql @@ -1,3 +1,8 @@ +{# + Complements is_pii_table (model-level) and should_disable_sampling_for_pii + (column-level) by adding test-level PII tag support. A test tagged with a PII + tag will have its samples disabled, consistent with the other two levels. +#} {% macro is_pii_test(flattened_test) %} {% if not elementary.get_config_var("disable_samples_on_pii_tags") %} {% do return(false) %} diff --git a/macros/edr/system/system_utils/should_show_sample_rows.sql b/macros/edr/system/system_utils/should_show_sample_rows.sql index 4b44a52e1..0ea682fb1 100644 --- a/macros/edr/system/system_utils/should_show_sample_rows.sql +++ b/macros/edr/system/system_utils/should_show_sample_rows.sql @@ -1,3 +1,16 @@ +{# + Inverse of PII protection: when enable_samples_on_show_sample_rows_tags is true, + samples are hidden by default and only shown when the show_sample_rows tag is present. + + Checks three levels in order: model → test → column (test's target column only). + Returns true if any level has a matching show_sample_rows tag. + + PII precedence: if disable_samples_on_pii_tags is also enabled and the same + model/column has a PII tag, PII wins and this returns false. This handles the + edge case where both features are active simultaneously. + + All tag matching is case-insensitive (tags are normalized to lowercase). +#} {% macro should_show_sample_rows(flattened_test) %} {% if not elementary.get_config_var("enable_samples_on_show_sample_rows_tags") %} {% do return(false) %} @@ -8,7 +21,10 @@ {% else %} {% set show_tags = (raw_show_tags or []) | map("lower") | list %} {% endif %} - {# Resolve PII tags once — only relevant when PII hiding is also enabled #} + {# + Resolve PII tags once upfront. We use `is string` (not `is iterable`) because + strings are iterable in Jinja — iterating a string gives individual characters. + #} {% set check_pii = elementary.get_config_var("disable_samples_on_pii_tags") %} {% if check_pii %} {% set raw_pii_tags = elementary.get_config_var("pii_tags") %} @@ -18,7 +34,7 @@ {% else %} {% set pii_tags = [] %} {% endif %} - {# Model-level check #} + {# Model-level: show_sample_rows on the model applies to all its tests #} {% set raw_model_tags = elementary.insensitive_get_dict_value( flattened_test, "model_tags", [] ) %} @@ -26,6 +42,7 @@ {% else %} {% set model_tags = (raw_model_tags or []) | map("lower") | list %} {% endif %} {% if elementary.lists_intersection(model_tags, show_tags) | length > 0 %} + {# PII on the model takes precedence over show_sample_rows on the same model #} {% if check_pii and elementary.lists_intersection( model_tags, pii_tags ) | length > 0 %} @@ -34,7 +51,7 @@ {% do return(true) %} {% endif %} - {# Test-level check #} + {# Test-level: show_sample_rows on the test definition itself #} {% set raw_test_tags = elementary.insensitive_get_dict_value( flattened_test, "tags", [] ) %} @@ -42,6 +59,7 @@ {% else %} {% set test_tags = (raw_test_tags or []) | map("lower") | list %} {% endif %} {% if elementary.lists_intersection(test_tags, show_tags) | length > 0 %} + {# If the model itself is PII-tagged, respect that even for test-level overrides #} {% if check_pii and elementary.lists_intersection( model_tags, pii_tags ) | length > 0 %} @@ -50,7 +68,10 @@ {% do return(true) %} {% endif %} - {# Column-level check: only the test's target column #} + {# + Column-level: only checks the specific column the test targets (test_column_name), + not all columns on the model. This avoids showing samples for unrelated columns. + #} {% set test_column_name = elementary.insensitive_get_dict_value( flattened_test, "test_column_name" ) %} @@ -67,6 +88,7 @@ {% if elementary.lists_intersection( col_tags, show_tags ) | length > 0 %} + {# PII on the column takes precedence over show_sample_rows on the same column #} {% if check_pii and elementary.lists_intersection( col_tags, pii_tags ) | length > 0 %} From 19f2367c099880338ab24c4f20e5d0b6afe455a1 Mon Sep 17 00:00:00 2001 From: Joost Boonzajer Flaes Date: Tue, 24 Mar 2026 12:26:57 +0200 Subject: [PATCH 10/12] fix: address review findings in show_sample_rows and PII macros - Restructure empty elif branch in test.sql with explicit comments - Column-level show_sample_rows now checks model-level PII tags too - Add explicit parens for Jinja operator precedence in get_pii_columns - Replace 'is iterable' with 'is string' guard in is_pii_table.sql (strings are iterable in Jinja, causing single tags to be split) Co-Authored-By: Claude Opus 4.6 (1M context) --- macros/edr/materializations/test/test.sql | 11 ++++++++--- .../get_pii_columns_from_parent_model.sql | 10 ++-------- macros/edr/system/system_utils/is_pii_table.sql | 16 ++++++---------- .../system_utils/should_show_sample_rows.sql | 15 ++++++++------- 4 files changed, 24 insertions(+), 28 deletions(-) diff --git a/macros/edr/materializations/test/test.sql b/macros/edr/materializations/test/test.sql index 681492c5c..0e41c2e99 100644 --- a/macros/edr/materializations/test/test.sql +++ b/macros/edr/materializations/test/test.sql @@ -86,12 +86,17 @@ 4. PII tag detection (model/test/column) — hide when disable_samples_on_pii_tags is true and a PII tag is detected at any level. #} - {% if disable_test_samples %} {% set sample_limit = 0 %} + {% if disable_test_samples %} + {% set sample_limit = 0 %} {% elif elementary.should_show_sample_rows(flattened_test) %} + {# Tag explicitly opts in — keep sample_limit as-is #} {% elif elementary.get_config_var("enable_samples_on_show_sample_rows_tags") %} + {# Feature is on but no show_sample_rows tag found — hide by default #} + {% set sample_limit = 0 %} + {% elif elementary.is_pii_table(flattened_test) %} + {% set sample_limit = 0 %} + {% elif elementary.is_pii_test(flattened_test) %} {% set sample_limit = 0 %} - {% elif elementary.is_pii_table(flattened_test) %} {% set sample_limit = 0 %} - {% elif elementary.is_pii_test(flattened_test) %} {% set sample_limit = 0 %} {% elif elementary.should_disable_sampling_for_pii(flattened_test) %} {% set sample_limit = 0 %} {% endif %} diff --git a/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql b/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql index cb3e444a1..30509119e 100644 --- a/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql +++ b/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql @@ -58,16 +58,10 @@ {# Skip column from PII list only if show_sample_rows is set and pii is not #} {% set has_show_tag = ( enable_show_tags - and elementary.lists_intersection( - all_column_tags_lower, show_tags - ) - | length - > 0 + and (elementary.lists_intersection(all_column_tags_lower, show_tags) | length > 0) ) %} {% set has_pii_tag = ( - elementary.lists_intersection(all_column_tags_lower, pii_tags) - | length - > 0 + elementary.lists_intersection(all_column_tags_lower, pii_tags) | length > 0 ) %} {% if has_show_tag and not has_pii_tag %} {% continue %} {% endif %} diff --git a/macros/edr/system/system_utils/is_pii_table.sql b/macros/edr/system/system_utils/is_pii_table.sql index 97c8819c3..850038e49 100644 --- a/macros/edr/system/system_utils/is_pii_table.sql +++ b/macros/edr/system/system_utils/is_pii_table.sql @@ -5,20 +5,16 @@ {% if not disable_samples_on_pii_tags %} {% do return(false) %} {% endif %} {% set raw_pii_tags = elementary.get_config_var("pii_tags") %} - {% set pii_tags = ( - (raw_pii_tags if raw_pii_tags is iterable else [raw_pii_tags]) - | map("lower") - | list - ) %} + {% if raw_pii_tags is string %} {% set pii_tags = [raw_pii_tags | lower] %} + {% else %} {% set pii_tags = (raw_pii_tags or []) | map("lower") | list %} + {% endif %} {% set raw_model_tags = elementary.insensitive_get_dict_value( flattened_test, "model_tags", [] ) %} - {% set model_tags = ( - (raw_model_tags if raw_model_tags is iterable else [raw_model_tags]) - | map("lower") - | list - ) %} + {% if raw_model_tags is string %} {% set model_tags = [raw_model_tags | lower] %} + {% else %} {% set model_tags = (raw_model_tags or []) | map("lower") | list %} + {% endif %} {% set intersection = elementary.lists_intersection(model_tags, pii_tags) %} {% set is_pii = intersection | length > 0 %} diff --git a/macros/edr/system/system_utils/should_show_sample_rows.sql b/macros/edr/system/system_utils/should_show_sample_rows.sql index 0ea682fb1..610e39a01 100644 --- a/macros/edr/system/system_utils/should_show_sample_rows.sql +++ b/macros/edr/system/system_utils/should_show_sample_rows.sql @@ -5,9 +5,9 @@ Checks three levels in order: model → test → column (test's target column only). Returns true if any level has a matching show_sample_rows tag. - PII precedence: if disable_samples_on_pii_tags is also enabled and the same - model/column has a PII tag, PII wins and this returns false. This handles the - edge case where both features are active simultaneously. + PII precedence: if disable_samples_on_pii_tags is also enabled and the model + or column has a PII tag, PII wins and this returns false. A model-level PII + tag blocks show_sample_rows at every level (model, test, and column). All tag matching is case-insensitive (tags are normalized to lowercase). #} @@ -88,10 +88,11 @@ {% if elementary.lists_intersection( col_tags, show_tags ) | length > 0 %} - {# PII on the column takes precedence over show_sample_rows on the same column #} - {% if check_pii and elementary.lists_intersection( - col_tags, pii_tags - ) | length > 0 %} + {# PII on the column or model takes precedence over show_sample_rows #} + {% if check_pii and ( + elementary.lists_intersection(col_tags, pii_tags) | length > 0 + or elementary.lists_intersection(model_tags, pii_tags) | length > 0 + ) %} {% do return(false) %} {% endif %} {% do return(true) %} From 605bdd01a4fe5325a8b7bca7d00e2ad8bf82b8fe Mon Sep 17 00:00:00 2001 From: Joost Boonzajer Flaes Date: Tue, 24 Mar 2026 12:30:06 +0200 Subject: [PATCH 11/12] style: apply sqlfmt formatting Co-Authored-By: Claude Opus 4.6 (1M context) --- macros/edr/materializations/test/test.sql | 11 ++++------- .../get_pii_columns_from_parent_model.sql | 11 +++++++---- .../system/system_utils/should_show_sample_rows.sql | 10 ++++++++-- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/macros/edr/materializations/test/test.sql b/macros/edr/materializations/test/test.sql index 0e41c2e99..71fe21d5f 100644 --- a/macros/edr/materializations/test/test.sql +++ b/macros/edr/materializations/test/test.sql @@ -86,17 +86,14 @@ 4. PII tag detection (model/test/column) — hide when disable_samples_on_pii_tags is true and a PII tag is detected at any level. #} - {% if disable_test_samples %} - {% set sample_limit = 0 %} + {% if disable_test_samples %} {% set sample_limit = 0 %} {% elif elementary.should_show_sample_rows(flattened_test) %} - {# Tag explicitly opts in — keep sample_limit as-is #} + {# Tag explicitly opts in — keep sample_limit as-is #} {% elif elementary.get_config_var("enable_samples_on_show_sample_rows_tags") %} {# Feature is on but no show_sample_rows tag found — hide by default #} {% set sample_limit = 0 %} - {% elif elementary.is_pii_table(flattened_test) %} - {% set sample_limit = 0 %} - {% elif elementary.is_pii_test(flattened_test) %} - {% set sample_limit = 0 %} + {% elif elementary.is_pii_table(flattened_test) %} {% set sample_limit = 0 %} + {% elif elementary.is_pii_test(flattened_test) %} {% set sample_limit = 0 %} {% elif elementary.should_disable_sampling_for_pii(flattened_test) %} {% set sample_limit = 0 %} {% endif %} diff --git a/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql b/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql index 30509119e..0de2648bc 100644 --- a/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql +++ b/macros/edr/system/system_utils/get_pii_columns_from_parent_model.sql @@ -56,12 +56,15 @@ {% set all_column_tags_lower = elementary.get_column_tags(column_node) %} {# Skip column from PII list only if show_sample_rows is set and pii is not #} - {% set has_show_tag = ( - enable_show_tags - and (elementary.lists_intersection(all_column_tags_lower, show_tags) | length > 0) + {% set has_show_tag = enable_show_tags and ( + elementary.lists_intersection(all_column_tags_lower, show_tags) + | length + > 0 ) %} {% set has_pii_tag = ( - elementary.lists_intersection(all_column_tags_lower, pii_tags) | length > 0 + elementary.lists_intersection(all_column_tags_lower, pii_tags) + | length + > 0 ) %} {% if has_show_tag and not has_pii_tag %} {% continue %} {% endif %} diff --git a/macros/edr/system/system_utils/should_show_sample_rows.sql b/macros/edr/system/system_utils/should_show_sample_rows.sql index 610e39a01..5b84e38af 100644 --- a/macros/edr/system/system_utils/should_show_sample_rows.sql +++ b/macros/edr/system/system_utils/should_show_sample_rows.sql @@ -90,8 +90,14 @@ ) | length > 0 %} {# PII on the column or model takes precedence over show_sample_rows #} {% if check_pii and ( - elementary.lists_intersection(col_tags, pii_tags) | length > 0 - or elementary.lists_intersection(model_tags, pii_tags) | length > 0 + elementary.lists_intersection(col_tags, pii_tags) + | length + > 0 + or elementary.lists_intersection( + model_tags, pii_tags + ) + | length + > 0 ) %} {% do return(false) %} {% endif %} From d5fbeb3695ecb01a8862681ecb2c552c11bbfd75 Mon Sep 17 00:00:00 2001 From: Joost Boonzajer Flaes Date: Tue, 24 Mar 2026 14:49:40 +0200 Subject: [PATCH 12/12] fix: handle Vertica Decimal special values (Infinity/NaN) in query runner Vertica's adapter returns Decimal special values where as_tuple().exponent is a string ('F'/'n') instead of int, causing a TypeError on comparison. Cherry-picked from worktree-fix-dimension-alerts (0eecf7d3). Co-Authored-By: Claude Opus 4.6 (1M context) --- integration_tests/tests/adapter_query_runner.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/integration_tests/tests/adapter_query_runner.py b/integration_tests/tests/adapter_query_runner.py index 6ac9d96ff..c45cbb83a 100644 --- a/integration_tests/tests/adapter_query_runner.py +++ b/integration_tests/tests/adapter_query_runner.py @@ -52,9 +52,14 @@ def _serialize_value(val: Any) -> Any: * Everything else is returned unchanged. """ if isinstance(val, Decimal): - # Match the Jinja macro: normalize, then int or float + # Match the Jinja macro: normalize, then int or float. + # Note: for special values (Infinity, NaN), as_tuple().exponent is a + # string ('F' or 'n'), not an int — convert those directly to float. normalized = val.normalize() - if normalized.as_tuple().exponent >= 0: + exponent = normalized.as_tuple().exponent + if isinstance(exponent, str): + return float(normalized) + if exponent >= 0: return int(normalized) return float(normalized) if isinstance(val, (datetime, date, time)):