From 0e44f14de6b04131db4a48f2cf1b34a1a5b16ead Mon Sep 17 00:00:00 2001 From: anil Date: Tue, 24 Feb 2026 14:45:54 +0100 Subject: [PATCH 1/3] fix: wider spaces between leaderboard copy pipes to prevent hitting queue limits --- services/libs/tinybird/pipes/leaderboards_codebase_size.pipe | 2 +- services/libs/tinybird/pipes/leaderboards_commits.pipe | 2 +- services/libs/tinybird/pipes/leaderboards_forks.pipe | 2 +- services/libs/tinybird/pipes/leaderboards_issue_response.pipe | 2 +- services/libs/tinybird/pipes/leaderboards_members.pipe | 2 +- services/libs/tinybird/pipes/leaderboards_merge_time.pipe | 2 +- services/libs/tinybird/pipes/leaderboards_organizations.pipe | 2 +- .../libs/tinybird/pipes/leaderboards_package_downloads.pipe | 2 +- .../pipes/leaderboards_project_active_contributors.pipe | 2 +- .../pipes/leaderboards_project_active_organizations.pipe | 2 +- services/libs/tinybird/pipes/leaderboards_resolution_rate.pipe | 2 +- .../libs/tinybird/pipes/leaderboards_small_project_commit.pipe | 2 +- services/libs/tinybird/pipes/leaderboards_stars.pipe | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/services/libs/tinybird/pipes/leaderboards_codebase_size.pipe b/services/libs/tinybird/pipes/leaderboards_codebase_size.pipe index b470f1b34b..ab6019a165 100644 --- a/services/libs/tinybird/pipes/leaderboards_codebase_size.pipe +++ b/services/libs/tinybird/pipes/leaderboards_codebase_size.pipe @@ -52,4 +52,4 @@ SQL > TYPE COPY TARGET_DATASOURCE leaderboards_copy_ds COPY_MODE append -COPY_SCHEDULE 0 1 * * * +COPY_SCHEDULE 10 1 * * * diff --git a/services/libs/tinybird/pipes/leaderboards_commits.pipe b/services/libs/tinybird/pipes/leaderboards_commits.pipe index f8ccab2662..ce5a9bc6f2 100644 --- a/services/libs/tinybird/pipes/leaderboards_commits.pipe +++ b/services/libs/tinybird/pipes/leaderboards_commits.pipe @@ -72,4 +72,4 @@ SQL > TYPE COPY TARGET_DATASOURCE leaderboards_copy_ds COPY_MODE append -COPY_SCHEDULE 5 1 * * * +COPY_SCHEDULE 20 1 * * * diff --git a/services/libs/tinybird/pipes/leaderboards_forks.pipe b/services/libs/tinybird/pipes/leaderboards_forks.pipe index 92f5ae2876..5d07e1aac8 100644 --- a/services/libs/tinybird/pipes/leaderboards_forks.pipe +++ b/services/libs/tinybird/pipes/leaderboards_forks.pipe @@ -50,4 +50,4 @@ SQL > TYPE COPY TARGET_DATASOURCE leaderboards_copy_ds COPY_MODE append -COPY_SCHEDULE 5 1 * * * +COPY_SCHEDULE 30 1 * * * diff --git a/services/libs/tinybird/pipes/leaderboards_issue_response.pipe b/services/libs/tinybird/pipes/leaderboards_issue_response.pipe index c17b0e9a43..49101d6785 100644 --- a/services/libs/tinybird/pipes/leaderboards_issue_response.pipe +++ b/services/libs/tinybird/pipes/leaderboards_issue_response.pipe @@ -73,4 +73,4 @@ SQL > TYPE COPY TARGET_DATASOURCE leaderboards_copy_ds COPY_MODE append -COPY_SCHEDULE 10 1 * * * +COPY_SCHEDULE 40 1 * * * diff --git a/services/libs/tinybird/pipes/leaderboards_members.pipe b/services/libs/tinybird/pipes/leaderboards_members.pipe index d8a91ae4f8..4684426457 100644 --- a/services/libs/tinybird/pipes/leaderboards_members.pipe +++ b/services/libs/tinybird/pipes/leaderboards_members.pipe @@ -91,4 +91,4 @@ SQL > TYPE COPY TARGET_DATASOURCE leaderboards_copy_ds COPY_MODE append -COPY_SCHEDULE 10 1 * * * +COPY_SCHEDULE 50 1 * * * diff --git a/services/libs/tinybird/pipes/leaderboards_merge_time.pipe b/services/libs/tinybird/pipes/leaderboards_merge_time.pipe index 744aaef7c3..b4e403f565 100644 --- a/services/libs/tinybird/pipes/leaderboards_merge_time.pipe +++ b/services/libs/tinybird/pipes/leaderboards_merge_time.pipe @@ -68,4 +68,4 @@ SQL > TYPE COPY TARGET_DATASOURCE leaderboards_copy_ds COPY_MODE append -COPY_SCHEDULE 15 1 * * * +COPY_SCHEDULE 0 2 * * * diff --git a/services/libs/tinybird/pipes/leaderboards_organizations.pipe b/services/libs/tinybird/pipes/leaderboards_organizations.pipe index 0a93eacdde..70c23910a9 100644 --- a/services/libs/tinybird/pipes/leaderboards_organizations.pipe +++ b/services/libs/tinybird/pipes/leaderboards_organizations.pipe @@ -90,4 +90,4 @@ SQL > TYPE COPY TARGET_DATASOURCE leaderboards_copy_ds COPY_MODE append -COPY_SCHEDULE 15 1 * * * +COPY_SCHEDULE 10 2 * * * diff --git a/services/libs/tinybird/pipes/leaderboards_package_downloads.pipe b/services/libs/tinybird/pipes/leaderboards_package_downloads.pipe index 8f45d509dd..493524f477 100644 --- a/services/libs/tinybird/pipes/leaderboards_package_downloads.pipe +++ b/services/libs/tinybird/pipes/leaderboards_package_downloads.pipe @@ -95,4 +95,4 @@ SQL > TYPE COPY TARGET_DATASOURCE leaderboards_copy_ds COPY_MODE append -COPY_SCHEDULE 20 1 * * * +COPY_SCHEDULE 20 2 * * * diff --git a/services/libs/tinybird/pipes/leaderboards_project_active_contributors.pipe b/services/libs/tinybird/pipes/leaderboards_project_active_contributors.pipe index ad2d5f0ac8..9421639d30 100644 --- a/services/libs/tinybird/pipes/leaderboards_project_active_contributors.pipe +++ b/services/libs/tinybird/pipes/leaderboards_project_active_contributors.pipe @@ -77,4 +77,4 @@ SQL > TYPE COPY TARGET_DATASOURCE leaderboards_copy_ds COPY_MODE append -COPY_SCHEDULE 20 1 * * * +COPY_SCHEDULE 30 2 * * * diff --git a/services/libs/tinybird/pipes/leaderboards_project_active_organizations.pipe b/services/libs/tinybird/pipes/leaderboards_project_active_organizations.pipe index 9665b302c6..fdbc57d9fc 100644 --- a/services/libs/tinybird/pipes/leaderboards_project_active_organizations.pipe +++ b/services/libs/tinybird/pipes/leaderboards_project_active_organizations.pipe @@ -77,4 +77,4 @@ SQL > TYPE COPY TARGET_DATASOURCE leaderboards_copy_ds COPY_MODE append -COPY_SCHEDULE 25 1 * * * +COPY_SCHEDULE 40 2 * * * diff --git a/services/libs/tinybird/pipes/leaderboards_resolution_rate.pipe b/services/libs/tinybird/pipes/leaderboards_resolution_rate.pipe index dd94301fa7..81062fb55d 100644 --- a/services/libs/tinybird/pipes/leaderboards_resolution_rate.pipe +++ b/services/libs/tinybird/pipes/leaderboards_resolution_rate.pipe @@ -47,4 +47,4 @@ SQL > TYPE COPY TARGET_DATASOURCE leaderboards_copy_ds COPY_MODE append -COPY_SCHEDULE 25 1 * * * +COPY_SCHEDULE 50 2 * * * diff --git a/services/libs/tinybird/pipes/leaderboards_small_project_commit.pipe b/services/libs/tinybird/pipes/leaderboards_small_project_commit.pipe index 48e4541014..739c983600 100644 --- a/services/libs/tinybird/pipes/leaderboards_small_project_commit.pipe +++ b/services/libs/tinybird/pipes/leaderboards_small_project_commit.pipe @@ -52,4 +52,4 @@ SQL > TYPE COPY TARGET_DATASOURCE leaderboards_copy_ds COPY_MODE append -COPY_SCHEDULE 30 1 * * * +COPY_SCHEDULE 0 3 * * * diff --git a/services/libs/tinybird/pipes/leaderboards_stars.pipe b/services/libs/tinybird/pipes/leaderboards_stars.pipe index 9c6f4e7aa3..4a95eae5a9 100644 --- a/services/libs/tinybird/pipes/leaderboards_stars.pipe +++ b/services/libs/tinybird/pipes/leaderboards_stars.pipe @@ -50,4 +50,4 @@ SQL > TYPE COPY TARGET_DATASOURCE leaderboards_copy_ds COPY_MODE append -COPY_SCHEDULE 30 1 * * * +COPY_SCHEDULE 10 3 * * * From bc928fa44020fbc8765a0cbb3fc6b68ceddd1a62 Mon Sep 17 00:00:00 2001 From: anil Date: Tue, 24 Feb 2026 15:14:01 +0100 Subject: [PATCH 2/3] feat: a push utility to push multiple pipes with match conditions --- services/libs/tinybird/scripts/push.sh | 64 ++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100755 services/libs/tinybird/scripts/push.sh diff --git a/services/libs/tinybird/scripts/push.sh b/services/libs/tinybird/scripts/push.sh new file mode 100755 index 0000000000..7ccbe734a3 --- /dev/null +++ b/services/libs/tinybird/scripts/push.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +PIPES_FOLDER="pipes" + +cd "$(dirname "$0")/.." || exit 1 + +show_help() { + cat << EOF +Usage: push.sh [OPTIONS] + +Push Tinybird pipe files using "tb push". + +OPTIONS: + --help Show this help message and exit + --match PATTERN Match files containing PATTERN in their name (wildcard match) + +EXAMPLES: + push.sh + Push all pipe files + + push.sh --match leaderboards_ + Push only files containing "leaderboards_" in their name + +EOF + exit 0 +} + +# Parse command line arguments +MATCH_PATTERN="" + +while [[ $# -gt 0 ]]; do + case $1 in + --help|-h) + show_help + ;; + --match) + if [ -z "$2" ] || [[ "$2" == --* ]]; then + echo "Error: --match requires a pattern argument" + echo "" + show_help + fi + MATCH_PATTERN="$2" + shift 2 + ;; + *) + echo "Error: Unknown option: $1" + echo "" + show_help + ;; + esac +done + +for file in "$PIPES_FOLDER"/*; do + if [ -f "$file" ]; then + local_basename=$(basename "$file") + + # Skip file if pattern is set and doesn't match + if [ -n "$MATCH_PATTERN" ] && [[ ! "$local_basename" == *$MATCH_PATTERN* ]]; then + continue + fi + + tb push "pipes/$local_basename" --force --yes + fi +done From 0746782c60af893b96845881ee0191d5206534fe Mon Sep 17 00:00:00 2001 From: anil Date: Tue, 24 Feb 2026 16:00:53 +0100 Subject: [PATCH 3/3] refactor: push script can be run from any directory --- services/libs/tinybird/scripts/push.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/libs/tinybird/scripts/push.sh b/services/libs/tinybird/scripts/push.sh index 7ccbe734a3..8e62b24c72 100755 --- a/services/libs/tinybird/scripts/push.sh +++ b/services/libs/tinybird/scripts/push.sh @@ -2,7 +2,7 @@ PIPES_FOLDER="pipes" -cd "$(dirname "$0")/.." || exit 1 +cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." || exit 1 show_help() { cat << EOF @@ -59,6 +59,6 @@ for file in "$PIPES_FOLDER"/*; do continue fi - tb push "pipes/$local_basename" --force --yes + tb push "$PIPES_FOLDER/$local_basename" --force --yes fi done