diff --git a/.github/workflows/test-acceptance.yml b/.github/workflows/test-acceptance.yml index 5e44af4..08f8218 100644 --- a/.github/workflows/test-acceptance.yml +++ b/.github/workflows/test-acceptance.yml @@ -25,6 +25,7 @@ jobs: env: ACCEPTANCE_SLICE: ${{ matrix.slice }} HOOKDECK_CLI_TESTING_API_KEY: ${{ secrets[matrix.api_key_secret] }} + HOOKDECK_CLI_TELEMETRY_DISABLED: "1" steps: - name: Check out code uses: actions/checkout@v3 @@ -36,3 +37,25 @@ jobs: - name: Run Go Acceptance Tests (slice ${{ matrix.slice }}) run: go test -tags="${{ matrix.tags }}" ./test/acceptance/... -v -timeout 12m + + # Telemetry proxy tests require the real telemetry header; matrix jobs set + # HOOKDECK_CLI_TELEMETRY_DISABLED=1. This job runs -tags=telemetry with + # HOOKDECK_CLI_TELEMETRY_DISABLED=0 so telemetry is on even if the repo sets a global opt-out. + acceptance-telemetry: + runs-on: ubuntu-latest + env: + ACCEPTANCE_SLICE: "0" + HOOKDECK_CLI_TESTING_API_KEY: ${{ secrets.HOOKDECK_CLI_TESTING_API_KEY }} + # Explicitly allow telemetry: repo/org env may set HOOKDECK_CLI_TELEMETRY_DISABLED=1 globally. + HOOKDECK_CLI_TELEMETRY_DISABLED: "0" + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: "1.24.9" + + - name: Run telemetry acceptance tests + run: go test -tags=telemetry ./test/acceptance/... -v -timeout 12m diff --git a/AGENTS.md b/AGENTS.md index c4f6473..99faed8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -274,6 +274,10 @@ go run cmd/hookdeck/main.go login --help - **Always run tests** when changing code: unit tests (`go test ./pkg/...`) and, for CLI-facing changes, acceptance tests (`go test ./test/acceptance/...`). Do not skip tests to avoid failures. - When running commands (build, test, acceptance tests), if you see **TLS/certificate errors** (e.g. `x509: certificate verify failed`, `tls: failed to verify certificate`), **permission errors** (e.g. `operation not permitted` when writing to the Go module cache), or similar failures that look environment-related, the command is likely running inside a **sandbox**. **Prompt the user** and **re-run the command with elevated permissions** (e.g. `required_permissions: ["network"]` for tests that need API access, or `["all"]` to disable the sandbox) so the operation can succeed. Do not treat a build or test as passed if stderr shows these errors, even when the process exit code is 0. +### GitHub CLI (`gh`) +- Use the **[GitHub CLI](https://cli.github.com/) (`gh`)** to read GitHub data and perform actions from the shell: **workflow runs and job logs** (e.g. `gh run list`, `gh run view --log-failed`, `gh run view --job --log`), **PRs and checks** (`gh pr view`, `gh pr checks`, `gh pr diff`), **API access** (`gh api`), and creating or updating PRs, issues, and releases. +- Install and authenticate `gh` where needed (e.g. `gh auth login`). If `gh` fails with TLS, network, or permission errors, re-run with **network** or **all** permissions when the agent sandbox may be blocking access. + ## 6. Documentation Standards ### Command help text (Short and Long) @@ -355,7 +359,7 @@ if apiErr, ok := err.(*hookdeck.APIError); ok { Acceptance tests require a Hookdeck API key. See [`test/acceptance/README.md`](test/acceptance/README.md) for full details. Quick setup: create `test/acceptance/.env` with `HOOKDECK_CLI_TESTING_API_KEY=`. The `.env` file is git-ignored and must never be committed. ### Acceptance tests and feature tags -Acceptance tests in `test/acceptance/` are partitioned by **feature build tags** so they can run in parallel (two slices in CI and locally). Each test file must have exactly one feature tag (e.g. `//go:build connection`, `//go:build request`). The runner (CI workflow or `run_parallel.sh`) maps features to slices and passes the corresponding `-tags="..."`; see [test/acceptance/README.md](test/acceptance/README.md) for slice mapping and setup. **Every new acceptance test file must have a feature tag**; otherwise it is included in every build and runs in both slices (duplicated). Use tags to balance and parallelize; same commands and env for local and CI. +Acceptance tests in `test/acceptance/` are partitioned by **feature build tags** so they can run in parallel (matrix slices plus a separate `acceptance-telemetry` job in CI; see [test/acceptance/README.md](test/acceptance/README.md)). Each `*_test.go` file must have exactly one feature tag (e.g. `//go:build connection`, `//go:build request`, `//go:build telemetry`). **Untagged test files are included in every `-tags=...` build**, including `-tags=telemetry` only, so non-telemetry tests would run in the telemetry job—do not add untagged `*_test.go` files. Use tags to balance and parallelize; same commands and env for local and CI. ### Unit Testing - Test validation logic thoroughly diff --git a/REFERENCE.md b/REFERENCE.md index 046f73f..4c268fe 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -59,12 +59,14 @@ hookdeck login [flags] | Flag | Type | Description | |------|------|-------------| | `-i, --interactive` | `bool` | Run interactive configuration mode if you cannot open a browser | +| `--local` | `bool` | Save credentials to current directory (.hookdeck/config.toml) | **Examples:** ```bash $ hookdeck login $ hookdeck login -i # interactive mode (no browser) +$ hookdeck login --local # save credentials to .hookdeck/config.toml ``` ## Logout @@ -103,8 +105,6 @@ hookdeck whoami ```bash $ hookdeck whoami ``` - -Output includes the current project type (Gateway, Outpost, or Console). ## Projects @@ -114,7 +114,7 @@ Output includes the current project type (Gateway, Outpost, or Console). ### hookdeck project list -List and filter projects by organization and project name substrings. Output shows project type (Gateway, Outpost, Console). Outbound projects are excluded from the list. +List and filter projects by organization and project name substrings **Usage:** @@ -126,8 +126,8 @@ hookdeck project list [] [] [flags] | Flag | Type | Description | |------|------|-------------| -| `--output` | `string` | Output format: `json` for machine-readable list (id, org, project, type, current) | -| `--type` | `string` | Filter by project type: `gateway`, `outpost`, or `console` | +| `--output` | `string` | Output format: json | +| `--type` | `string` | Filter by project type: gateway, outpost, console | **Examples:** @@ -135,7 +135,6 @@ hookdeck project list [] [] [flags] $ hookdeck project list Acme / Ecommerce Production (current) | Gateway Acme / Ecommerce Staging | Gateway - $ hookdeck project list --output json $ hookdeck project list --type gateway ``` @@ -161,14 +160,11 @@ hookdeck project use [ []] [flags] $ hookdeck project use Use the arrow keys to navigate: ↓ ↑ → ← ? Select Project: -▸ [Acme] Ecommerce Production -[Acme] Ecommerce Staging -[Acme] Ecommerce Development - -Selecting project [Acme] Ecommerce Staging +▸ Acme / Ecommerce Production (current) | Gateway +Acme / Ecommerce Staging | Gateway $ hookdeck project use --local -Pinning project [Acme] Ecommerce Staging to current directory +Pinning project to current directory ``` ## Local Development @@ -219,7 +215,6 @@ Commands for managing Event Gateway sources, destinations, connections, transformations, events, requests, metrics, and MCP server. The gateway command group provides full access to all Event Gateway resources. -**Gateway commands require the current project to be a Gateway project** (inbound or console). If your project type is Outpost or you have no project selected, run `hookdeck project use` to switch to a Gateway project first. **Usage:** @@ -282,22 +277,22 @@ hookdeck gateway connection list [flags] ```bash # List all connections -hookdeck connection list +hookdeck gateway connection list # Filter by connection name -hookdeck connection list --name my-connection +hookdeck gateway connection list --name my-connection # Filter by source ID -hookdeck connection list --source-id src_abc123 +hookdeck gateway connection list --source-id src_abc123 # Filter by destination ID -hookdeck connection list --destination-id dst_def456 +hookdeck gateway connection list --destination-id dst_def456 # Include disabled connections -hookdeck connection list --disabled +hookdeck gateway connection list --disabled # Limit results -hookdeck connection list --limit 10 +hookdeck gateway connection list --limit 10 ``` ### hookdeck gateway connection create @@ -353,10 +348,10 @@ hookdeck gateway connection create [flags] | `--rule-deduplicate-include-fields` | `string` | Comma-separated list of fields to include for deduplication | | `--rule-deduplicate-window` | `int` | Time window in seconds for deduplication (default "0") | | `--rule-delay` | `int` | Delay in milliseconds (default "0") | -| `--rule-filter-body` | `string` | JQ expression to filter on request body | -| `--rule-filter-headers` | `string` | JQ expression to filter on request headers | -| `--rule-filter-path` | `string` | JQ expression to filter on request path | -| `--rule-filter-query` | `string` | JQ expression to filter on request query parameters | +| `--rule-filter-body` | `string` | Filter on request body using Hookdeck filter syntax (JSON) | +| `--rule-filter-headers` | `string` | Filter on request headers using Hookdeck filter syntax (JSON) | +| `--rule-filter-path` | `string` | Filter on request path using Hookdeck filter syntax (JSON) | +| `--rule-filter-query` | `string` | Filter on request query parameters using Hookdeck filter syntax (JSON) | | `--rule-retry-count` | `int` | Number of retry attempts (default "0") | | `--rule-retry-interval` | `int` | Interval between retries in milliseconds (default "0") | | `--rule-retry-response-status-codes` | `string` | Comma-separated HTTP status codes to retry on | @@ -386,19 +381,19 @@ hookdeck gateway connection create [flags] ```bash # Create with inline source and destination -hookdeck connection create \ +hookdeck gateway connection create \ --name "test-webhooks-to-local" \ --source-type WEBHOOK --source-name "test-webhooks" \ --destination-type CLI --destination-name "local-dev" # Create with existing resources -hookdeck connection create \ +hookdeck gateway connection create \ --name "github-to-api" \ --source-id src_abc123 \ --destination-id dst_def456 # Create with source configuration options -hookdeck connection create \ +hookdeck gateway connection create \ --name "api-webhooks" \ --source-type WEBHOOK --source-name "api-source" \ --source-allowed-http-methods "POST,PUT,PATCH" \ @@ -418,6 +413,12 @@ You can specify either a connection ID or name. hookdeck gateway connection get [flags] ``` +**Arguments:** + +| Argument | Type | Description | +|----------|------|-------------| +| `connection-id-or-name` | `string` | **Required.** Connection ID or name | + **Flags:** | Flag | Type | Description | @@ -430,10 +431,10 @@ hookdeck gateway connection get [flags] ```bash # Get connection by ID -hookdeck connection get conn_abc123 +hookdeck gateway connection get conn_abc123 # Get connection by name -hookdeck connection get my-connection +hookdeck gateway connection get my-connection ``` ### hookdeck gateway connection update @@ -448,6 +449,12 @@ and allows changing any field including the connection name. hookdeck gateway connection update [flags] ``` +**Arguments:** + +| Argument | Type | Description | +|----------|------|-------------| +| `connection-id` | `string` | **Required.** Connection ID | + **Flags:** | Flag | Type | Description | @@ -460,10 +467,10 @@ hookdeck gateway connection update [flags] | `--rule-deduplicate-include-fields` | `string` | Comma-separated list of fields to include for deduplication | | `--rule-deduplicate-window` | `int` | Time window in seconds for deduplication (default "0") | | `--rule-delay` | `int` | Delay in milliseconds (default "0") | -| `--rule-filter-body` | `string` | JQ expression to filter on request body | -| `--rule-filter-headers` | `string` | JQ expression to filter on request headers | -| `--rule-filter-path` | `string` | JQ expression to filter on request path | -| `--rule-filter-query` | `string` | JQ expression to filter on request query parameters | +| `--rule-filter-body` | `string` | Filter on request body using Hookdeck filter syntax (JSON) | +| `--rule-filter-headers` | `string` | Filter on request headers using Hookdeck filter syntax (JSON) | +| `--rule-filter-path` | `string` | Filter on request path using Hookdeck filter syntax (JSON) | +| `--rule-filter-query` | `string` | Filter on request query parameters using Hookdeck filter syntax (JSON) | | `--rule-retry-count` | `int` | Number of retry attempts (default "0") | | `--rule-retry-interval` | `int` | Interval between retries in milliseconds (default "0") | | `--rule-retry-response-status-codes` | `string` | Comma-separated HTTP status codes to retry on | @@ -504,6 +511,12 @@ Delete a connection. hookdeck gateway connection delete [flags] ``` +**Arguments:** + +| Argument | Type | Description | +|----------|------|-------------| +| `connection-id` | `string` | **Required.** Connection ID | + **Flags:** | Flag | Type | Description | @@ -514,10 +527,10 @@ hookdeck gateway connection delete [flags] ```bash # Delete a connection (with confirmation) -hookdeck connection delete conn_abc123 +hookdeck gateway connection delete conn_abc123 # Force delete without confirmation -hookdeck connection delete conn_abc123 --force +hookdeck gateway connection delete conn_abc123 --force ``` ### hookdeck gateway connection upsert @@ -542,6 +555,12 @@ Create a new connection or update an existing one by name (idempotent). hookdeck gateway connection upsert [flags] ``` +**Arguments:** + +| Argument | Type | Description | +|----------|------|-------------| +| `name` | `string` | **Required.** Connection name (create or update by name) | + **Flags:** | Flag | Type | Description | @@ -584,10 +603,10 @@ hookdeck gateway connection upsert [flags] | `--rule-deduplicate-include-fields` | `string` | Comma-separated list of fields to include for deduplication | | `--rule-deduplicate-window` | `int` | Time window in seconds for deduplication (default "0") | | `--rule-delay` | `int` | Delay in milliseconds (default "0") | -| `--rule-filter-body` | `string` | JQ expression to filter on request body | -| `--rule-filter-headers` | `string` | JQ expression to filter on request headers | -| `--rule-filter-path` | `string` | JQ expression to filter on request path | -| `--rule-filter-query` | `string` | JQ expression to filter on request query parameters | +| `--rule-filter-body` | `string` | Filter on request body using Hookdeck filter syntax (JSON) | +| `--rule-filter-headers` | `string` | Filter on request headers using Hookdeck filter syntax (JSON) | +| `--rule-filter-path` | `string` | Filter on request path using Hookdeck filter syntax (JSON) | +| `--rule-filter-query` | `string` | Filter on request query parameters using Hookdeck filter syntax (JSON) | | `--rule-retry-count` | `int` | Number of retry attempts (default "0") | | `--rule-retry-interval` | `int` | Interval between retries in milliseconds (default "0") | | `--rule-retry-response-status-codes` | `string` | Comma-separated HTTP status codes to retry on | @@ -617,22 +636,22 @@ hookdeck gateway connection upsert [flags] ```bash # Create or update a connection with inline source and destination -hookdeck connection upsert "my-connection" \ +hookdeck gateway connection upsert "my-connection" \ --source-name "stripe-prod" --source-type STRIPE \ --destination-name "my-api" --destination-type HTTP --destination-url https://api.example.com # Update just the rate limit on an existing connection -hookdeck connection upsert my-connection \ +hookdeck gateway connection upsert my-connection \ --destination-rate-limit 100 --destination-rate-limit-period minute # Update source configuration options -hookdeck connection upsert my-connection \ +hookdeck gateway connection upsert my-connection \ --source-allowed-http-methods "POST,PUT,DELETE" \ --source-custom-response-content-type "json" \ --source-custom-response-body '{"status":"received"}' # Preview changes without applying them -hookdeck connection upsert my-connection \ +hookdeck gateway connection upsert my-connection \ --destination-rate-limit 200 --destination-rate-limit-period hour \ --dry-run ``` @@ -645,6 +664,12 @@ Enable a disabled connection. ```bash hookdeck gateway connection enable ``` + +**Arguments:** + +| Argument | Type | Description | +|----------|------|-------------| +| `connection-id` | `string` | **Required.** Connection ID | ### hookdeck gateway connection disable Disable an active connection. It will stop receiving new events until re-enabled. @@ -654,6 +679,12 @@ Disable an active connection. It will stop receiving new events until re-enabled ```bash hookdeck gateway connection disable ``` + +**Arguments:** + +| Argument | Type | Description | +|----------|------|-------------| +| `connection-id` | `string` | **Required.** Connection ID | ### hookdeck gateway connection pause Pause a connection temporarily. @@ -665,6 +696,12 @@ The connection will queue incoming events until unpaused. ```bash hookdeck gateway connection pause ``` + +**Arguments:** + +| Argument | Type | Description | +|----------|------|-------------| +| `connection-id` | `string` | **Required.** Connection ID | ### hookdeck gateway connection unpause Resume a paused connection. @@ -676,6 +713,12 @@ The connection will start processing queued events. ```bash hookdeck gateway connection unpause ``` + +**Arguments:** + +| Argument | Type | Description | +|----------|------|-------------| +| `connection-id` | `string` | **Required.** Connection ID | ## Sources @@ -1864,6 +1907,7 @@ hookdeck ci [flags] | Flag | Type | Description | |------|------|-------------| | `--api-key` | `string` | Your Hookdeck Project API key. The CLI reads from HOOKDECK_API_KEY if not provided. | +| `--local` | `bool` | Save credentials to current directory (.hookdeck/config.toml) | | `--name` | `string` | Name of the CI run (ex: GITHUB_REF) for identification in the dashboard | **Examples:** diff --git a/pkg/cmd/connection_create.go b/pkg/cmd/connection_create.go index f5921df..a3cfd50 100644 --- a/pkg/cmd/connection_create.go +++ b/pkg/cmd/connection_create.go @@ -112,19 +112,19 @@ func newConnectionCreateCmd() *connectionCreateCmd { Examples: # Create with inline source and destination - hookdeck connection create \ + hookdeck gateway connection create \ --name "test-webhooks-to-local" \ --source-type WEBHOOK --source-name "test-webhooks" \ --destination-type CLI --destination-name "local-dev" # Create with existing resources - hookdeck connection create \ + hookdeck gateway connection create \ --name "github-to-api" \ --source-id src_abc123 \ --destination-id dst_def456 # Create with source configuration options - hookdeck connection create \ + hookdeck gateway connection create \ --name "api-webhooks" \ --source-type WEBHOOK --source-name "api-source" \ --source-allowed-http-methods "POST,PUT,PATCH" \ diff --git a/pkg/cmd/connection_delete.go b/pkg/cmd/connection_delete.go index 826c668..4201c29 100644 --- a/pkg/cmd/connection_delete.go +++ b/pkg/cmd/connection_delete.go @@ -26,10 +26,10 @@ func newConnectionDeleteCmd() *connectionDeleteCmd { Examples: # Delete a connection (with confirmation) - hookdeck connection delete conn_abc123 + hookdeck gateway connection delete conn_abc123 # Force delete without confirmation - hookdeck connection delete conn_abc123 --force`, + hookdeck gateway connection delete conn_abc123 --force`, PreRunE: cc.validateFlags, RunE: cc.runConnectionDeleteCmd, } diff --git a/pkg/cmd/connection_get.go b/pkg/cmd/connection_get.go index 9564430..2b580fd 100644 --- a/pkg/cmd/connection_get.go +++ b/pkg/cmd/connection_get.go @@ -33,10 +33,10 @@ func newConnectionGetCmd() *connectionGetCmd { Examples: # Get connection by ID - hookdeck connection get conn_abc123 + hookdeck gateway connection get conn_abc123 # Get connection by name - hookdeck connection get my-connection`, + hookdeck gateway connection get my-connection`, RunE: cc.runConnectionGetCmd, } cc.cmd.Annotations = map[string]string{ diff --git a/pkg/cmd/connection_list.go b/pkg/cmd/connection_list.go index f2e19d9..5cfb2ff 100644 --- a/pkg/cmd/connection_list.go +++ b/pkg/cmd/connection_list.go @@ -34,22 +34,22 @@ func newConnectionListCmd() *connectionListCmd { Examples: # List all connections - hookdeck connection list + hookdeck gateway connection list # Filter by connection name - hookdeck connection list --name my-connection + hookdeck gateway connection list --name my-connection # Filter by source ID - hookdeck connection list --source-id src_abc123 + hookdeck gateway connection list --source-id src_abc123 # Filter by destination ID - hookdeck connection list --destination-id dst_def456 + hookdeck gateway connection list --destination-id dst_def456 # Include disabled connections - hookdeck connection list --disabled + hookdeck gateway connection list --disabled # Limit results - hookdeck connection list --limit 10`, + hookdeck gateway connection list --limit 10`, RunE: cc.runConnectionListCmd, } diff --git a/pkg/cmd/connection_upsert.go b/pkg/cmd/connection_upsert.go index 622cfe3..3186b31 100644 --- a/pkg/cmd/connection_upsert.go +++ b/pkg/cmd/connection_upsert.go @@ -42,22 +42,22 @@ func newConnectionUpsertCmd() *connectionUpsertCmd { Examples: # Create or update a connection with inline source and destination - hookdeck connection upsert "my-connection" \ + hookdeck gateway connection upsert "my-connection" \ --source-name "stripe-prod" --source-type STRIPE \ --destination-name "my-api" --destination-type HTTP --destination-url https://api.example.com # Update just the rate limit on an existing connection - hookdeck connection upsert my-connection \ + hookdeck gateway connection upsert my-connection \ --destination-rate-limit 100 --destination-rate-limit-period minute # Update source configuration options - hookdeck connection upsert my-connection \ + hookdeck gateway connection upsert my-connection \ --source-allowed-http-methods "POST,PUT,DELETE" \ --source-custom-response-content-type "json" \ --source-custom-response-body '{"status":"received"}' # Preview changes without applying them - hookdeck connection upsert my-connection \ + hookdeck gateway connection upsert my-connection \ --destination-rate-limit 200 --destination-rate-limit-period hour \ --dry-run`, PreRunE: cu.validateUpsertFlags, diff --git a/pkg/cmd/root.go b/pkg/cmd/root.go index 4da3162..fe42067 100644 --- a/pkg/cmd/root.go +++ b/pkg/cmd/root.go @@ -51,7 +51,9 @@ func initTelemetry(cmd *cobra.Command) { tel.SetEnvironment(hookdeck.DetectEnvironment()) tel.SetCommandContext(cmd) tel.SetDeviceName(Config.DeviceName) - tel.SetInvocationID(hookdeck.NewInvocationID()) + if tel.InvocationID == "" { + tel.SetInvocationID(hookdeck.NewInvocationID()) + } } // RootCmd returns the root command for use by tools (e.g. generate-reference). diff --git a/pkg/cmd/telemetry_test.go b/pkg/cmd/telemetry_test.go index 5ee7ab7..f0a954e 100644 --- a/pkg/cmd/telemetry_test.go +++ b/pkg/cmd/telemetry_test.go @@ -86,6 +86,39 @@ func TestInitTelemetryResetBetweenCalls(t *testing.T) { require.NotEqual(t, id1, tel2.InvocationID) } +// TestInvocationIDPersistsAcrossMultipleInitTelemetryCalls reproduces the v2.0.0 bug: when +// initTelemetry is called multiple times in one process (e.g. root PreRun, then gateway PreRun, +// then connection PreRun), the invocation_id must stay the same so all API requests in that run +// share one ID. Without the fix, each initTelemetry call overwrites with a new ID; this test +// asserts the ID is unchanged after a second call. On v2.0.0 (repro branch) it fails. +func TestInvocationIDPersistsAcrossMultipleInitTelemetryCalls(t *testing.T) { + hookdeck.ResetTelemetryInstanceForTesting() + + root := &cobra.Command{Use: "hookdeck"} + gateway := &cobra.Command{Use: "gateway"} + connection := &cobra.Command{Use: "connection"} + root.AddCommand(gateway) + gateway.AddCommand(connection) + + Config.DeviceName = "test-device" + initTelemetry(root) + tel := hookdeck.GetTelemetryInstance() + firstID := tel.InvocationID + require.NotEmpty(t, firstID, "first initTelemetry must set invocation_id") + + // Second call simulates gateway PreRun; must not overwrite invocation_id. + initTelemetry(gateway) + tel = hookdeck.GetTelemetryInstance() + require.Equal(t, firstID, tel.InvocationID, + "invocation_id must persist across initTelemetry calls in the same process (fix: set only when empty)") + + // Third call simulates connection PreRun; must still not overwrite. + initTelemetry(connection) + tel = hookdeck.GetTelemetryInstance() + require.Equal(t, firstID, tel.InvocationID, + "invocation_id must persist after third initTelemetry call") +} + // TestInitTelemetryWhenDisabled verifies that initTelemetry always populates the // singleton (Source, CommandPath, etc.) even when telemetry is disabled. The // call must happen for every command; PerformRequest later skips sending the diff --git a/test/acceptance/README.md b/test/acceptance/README.md index 95280d1..cd0e6d6 100644 --- a/test/acceptance/README.md +++ b/test/acceptance/README.md @@ -18,6 +18,16 @@ These tests require browser-based authentication via `hookdeck login` and must b **Why Manual?** These tests access endpoints (like `/teams`) that require CLI authentication keys obtained through interactive browser login, which aren't available to CI service accounts. +### Transient HTTP 502 from the API + +`CLIRunner.Run`, `RunWithEnv`, and `RunFromCwd` retry the same command up to **4** times when combined stdout/stderr looks like a Hookdeck API **HTTP 502** (matching CLI error text such as `unexpected http status code: 502`). **503** and **504** are not treated specially. Each retry is logged with `t.Logf` (attempt number, command summary, output excerpts); if all attempts fail, a final log line notes that the run is giving up. + +### Recording proxy (telemetry tests) + +Some tests (e.g. `TestTelemetryGatewayConnectionListProxy` in `telemetry_test.go`, `TestTelemetryListenProxy` in `telemetry_listen_test.go`) use a **recording proxy**: the CLI is run with `--api-base` pointing at a local HTTP server that forwards every request to the real Hookdeck API and records method, path, and the `X-Hookdeck-CLI-Telemetry` header. The same `CLIRunner` and `go run main.go` flow are used as in other acceptance tests; only the API base URL is overridden so traffic goes through the proxy. This verifies that a single CLI run sends consistent telemetry (same `invocation_id` and `command_path`) on all API calls. Helpers: `StartRecordingProxy`, `AssertTelemetryConsistent`. + +**Login telemetry test (TestTelemetryLoginProxy)** uses the same proxy approach: it runs `hookdeck login --api-key KEY` with `--api-base` set to the proxy, and asserts exactly one recorded request (GET `/2025-07-01/cli-auth/validate`) with consistent telemetry. It uses **HOOKDECK_CLI_TESTING_CLI_KEY** (not the API/CI key), because the validate endpoint accepts CLI keys from interactive login; if unset, the test is skipped. Other telemetry tests still use the normal API key via `NewCLIRunner`. + ## Setup ### Local Development @@ -42,38 +52,45 @@ HOOKDECK_CLI_TESTING_API_KEY_3=key_for_slice2 ### CI/CD -CI runs acceptance tests in **three parallel jobs**, each with its own API key (`HOOKDECK_CLI_TESTING_API_KEY`, `HOOKDECK_CLI_TESTING_API_KEY_2`, `HOOKDECK_CLI_TESTING_API_KEY_3`). No test-name list in the workflow—tests are partitioned by **feature tags** (see [Parallelisation](#parallelisation)). +CI runs **three parallel matrix jobs**, each with its own API key (`HOOKDECK_CLI_TESTING_API_KEY`, `HOOKDECK_CLI_TESTING_API_KEY_2`, `HOOKDECK_CLI_TESTING_API_KEY_3`). Those jobs set **`HOOKDECK_CLI_TELEMETRY_DISABLED=1`** so the CLI does not send telemetry during normal acceptance tests. + +A **fourth job** (`acceptance-telemetry` in `.github/workflows/test-acceptance.yml`) sets **`HOOKDECK_CLI_TELEMETRY_DISABLED=0`** so the CLI sends `X-Hookdeck-CLI-Telemetry` even if the repository or organization defines `HOOKDECK_CLI_TELEMETRY_DISABLED=1` for other jobs. It runs `go test -tags=telemetry` only (all proxy tests that assert that header, including listen, live under the `telemetry` build tag). It uses `HOOKDECK_CLI_TESTING_API_KEY` and `ACCEPTANCE_SLICE=0` (same project as slice 0; tests use unique resource names). + +No test-name list in the workflow—tests are partitioned by **feature tags** (see [Parallelisation](#parallelisation)). ## Running Tests ### Run all automated tests (one key) Pass all feature tags so every automated test file is included: ```bash -go test -tags="basic connection source destination gateway mcp listen project_use connection_list connection_upsert connection_error_hints connection_oauth_aws connection_update request event attempt metrics issue transformation" ./test/acceptance/... -v +go test -tags="basic connection source destination gateway mcp listen project_use connection_list connection_upsert connection_error_hints connection_oauth_aws connection_update request event telemetry attempt metrics issue transformation" ./test/acceptance/... -v ``` ### Run one slice (for CI or local) Same commands as CI; use when debugging a subset or running in parallel: ```bash # Slice 0 (same tags as CI job 0) -ACCEPTANCE_SLICE=0 go test -tags="basic connection source destination gateway mcp listen project_use connection_list connection_upsert connection_error_hints connection_oauth_aws connection_update" ./test/acceptance/... -v -timeout 12m +ACCEPTANCE_SLICE=0 HOOKDECK_CLI_TELEMETRY_DISABLED=1 go test -tags="basic connection source destination gateway mcp listen project_use connection_list connection_upsert connection_error_hints connection_oauth_aws connection_update" ./test/acceptance/... -v -timeout 12m # Slice 1 (same tags as CI job 1) -ACCEPTANCE_SLICE=1 go test -tags="request event" ./test/acceptance/... -v -timeout 12m +ACCEPTANCE_SLICE=1 HOOKDECK_CLI_TELEMETRY_DISABLED=1 go test -tags="request event" ./test/acceptance/... -v -timeout 12m # Slice 2 (same tags as CI job 2) -ACCEPTANCE_SLICE=2 go test -tags="attempt metrics issue transformation" ./test/acceptance/... -v -timeout 12m +ACCEPTANCE_SLICE=2 HOOKDECK_CLI_TELEMETRY_DISABLED=1 go test -tags="attempt metrics issue transformation" ./test/acceptance/... -v -timeout 12m + +# Telemetry (same as CI acceptance-telemetry: force telemetry on) +ACCEPTANCE_SLICE=0 HOOKDECK_CLI_TELEMETRY_DISABLED=0 go test -tags=telemetry ./test/acceptance/... -v -timeout 12m ``` -For slice 1 set `HOOKDECK_CLI_TESTING_API_KEY_2`; for slice 2 set `HOOKDECK_CLI_TESTING_API_KEY_3` (or set `HOOKDECK_CLI_TESTING_API_KEY` to that key). +For slice 1 set `HOOKDECK_CLI_TESTING_API_KEY_2`; for slice 2 set `HOOKDECK_CLI_TESTING_API_KEY_3` (or set `HOOKDECK_CLI_TESTING_API_KEY` to that key). For telemetry, use the slice 0 key and set `HOOKDECK_CLI_TELEMETRY_DISABLED=0` (overrides a global opt-out). **Project list tests** (`TestProjectListShowsType`, `TestProjectListJSONOutput`) require a **CLI key**, not an API or CI key: only keys created via interactive login can list or switch projects. Set `HOOKDECK_CLI_TESTING_CLI_KEY` in your `.env` (or environment) to run these tests; if unset, they are skipped with a clear message. ### Run in parallel locally (three keys) -From the **repository root**, run the script that runs all three slices in parallel (same as CI): +From the **repository root**, run the script that runs three matrix slices plus telemetry in parallel (same as CI): ```bash ./test/acceptance/run_parallel.sh ``` -Requires `HOOKDECK_CLI_TESTING_API_KEY`, `HOOKDECK_CLI_TESTING_API_KEY_2`, and `HOOKDECK_CLI_TESTING_API_KEY_3` in `.env` or the environment. +Requires `HOOKDECK_CLI_TESTING_API_KEY`, `HOOKDECK_CLI_TESTING_API_KEY_2`, and `HOOKDECK_CLI_TESTING_API_KEY_3` in `.env` or the environment. The script sets `HOOKDECK_CLI_TELEMETRY_DISABLED=1` for matrix slices and `HOOKDECK_CLI_TELEMETRY_DISABLED=0` for the telemetry run (same as CI). ### Run manual tests (requires human authentication): ```bash @@ -93,15 +110,16 @@ Use the same `-tags` as "Run all" if you want to skip the full acceptance set. A ## Parallelisation -Tests are partitioned by **feature build tags** so CI and local runs can execute three slices in parallel (each slice uses its own Hookdeck project and config file). +Tests are partitioned by **feature build tags** so CI and local runs can execute three matrix slices in parallel (each slice uses its own Hookdeck project and config file). - **Slice 0 features:** `basic`, `connection`, `source`, `destination`, `gateway`, `mcp`, `listen`, `project_use`, `connection_list`, `connection_upsert`, `connection_error_hints`, `connection_oauth_aws`, `connection_update` - **Slice 1 features:** `request`, `event` - **Slice 2 features:** `attempt`, `metrics`, `issue`, `transformation` +- **Telemetry job:** `telemetry` only — separate CI job with telemetry **not** disabled (see [CI/CD](#cicd)) -The CI workflow (`.github/workflows/test-acceptance.yml`) runs three jobs with the same `go test -tags="..."` commands and env (`ACCEPTANCE_SLICE`, API keys). No test names or regexes are listed in YAML. +The CI workflow (`.github/workflows/test-acceptance.yml`) runs three matrix jobs plus `acceptance-telemetry`. Matrix jobs set `HOOKDECK_CLI_TELEMETRY_DISABLED=1`; the telemetry job does not. No test names or regexes are listed in YAML. -**Untagged files:** A test file with **no** build tag is included in every build and runs in **both** slices (duplicated). **Every new acceptance test file must have exactly one feature tag** so it runs in only one slice. +**Untagged files:** A test file with **no** build tag is included in **every** `go test -tags=...` build, including **`acceptance-telemetry`** (`-tags=telemetry` only), so non-telemetry tests would run there too. **Every new acceptance test file must have exactly one feature tag** so it runs in only one matrix slice and not in the telemetry job. ## Manual Test Workflow diff --git a/test/acceptance/connection_rules_json_test.go b/test/acceptance/connection_rules_json_test.go index 5d78382..e376e76 100644 --- a/test/acceptance/connection_rules_json_test.go +++ b/test/acceptance/connection_rules_json_test.go @@ -1,3 +1,5 @@ +//go:build connection + package acceptance import ( diff --git a/test/acceptance/destination_config_json_test.go b/test/acceptance/destination_config_json_test.go index 6837090..0fa2a44 100644 --- a/test/acceptance/destination_config_json_test.go +++ b/test/acceptance/destination_config_json_test.go @@ -1,3 +1,5 @@ +//go:build destination + package acceptance import ( diff --git a/test/acceptance/helpers.go b/test/acceptance/helpers.go index f38d5b0..d58553d 100644 --- a/test/acceptance/helpers.go +++ b/test/acceptance/helpers.go @@ -5,11 +5,15 @@ import ( "bytes" "encoding/json" "fmt" + "io" "net/http" + "net/http/httptest" + "net/url" "os" "os/exec" "path/filepath" "strings" + "sync" "testing" "time" @@ -17,6 +21,216 @@ import ( "github.com/stretchr/testify/require" ) +// defaultAPIUpstream is the real Hookdeck API base URL used by the recording proxy. +const defaultAPIUpstream = "https://api.hookdeck.com" + +// acceptance502MaxAttempts is how many times CLIRunner runs a command when the +// combined output looks like a Hookdeck API HTTP 502 (transient gateway errors). +const acceptance502MaxAttempts = 4 + +// acceptance502RetryDelay is the pause between 502 retries. +const acceptance502RetryDelay = 2 * time.Second + +// acceptance502LogExcerpt is the max chars of stdout/stderr to include in retry logs. +const acceptance502LogExcerpt = 800 + +// combinedOutputLooksLikeHTTP502 returns true when stdout+stderr match patterns +// emitted by the CLI/API client for HTTP 502 (only; 503/504 are not retried here). +func combinedOutputLooksLikeHTTP502(stdout, stderr string) bool { + combined := stdout + "\n" + stderr + return strings.Contains(combined, "status code: 502") || + strings.Contains(combined, "status=502") || + strings.Contains(combined, "error code: 502") +} + +func excerptFor502Log(s string, max int) string { + s = strings.TrimSpace(s) + if s == "" { + return "(empty)" + } + if len(s) <= max { + return s + } + return "…" + s[len(s)-max:] +} + +func commandSummaryFor502Log(args []string) string { + const max = 200 + s := strings.Join(args, " ") + if len(s) <= max { + return s + } + return s[:max] + "…" +} + +// runWithHTTP502Retry re-runs run() when the process exits with an error and +// output looks like HTTP 502 from the Hookdeck API. Logs each retry clearly via t.Logf. +func (r *CLIRunner) runWithHTTP502Retry(commandSummary string, run func() (stdout, stderr string, err error)) (stdout, stderr string, err error) { + r.t.Helper() + var lastStdout, lastStderr string + var lastErr error + for attempt := 1; attempt <= acceptance502MaxAttempts; attempt++ { + lastStdout, lastStderr, lastErr = run() + if lastErr == nil || !combinedOutputLooksLikeHTTP502(lastStdout, lastStderr) { + return lastStdout, lastStderr, lastErr + } + if attempt < acceptance502MaxAttempts { + r.t.Logf("acceptance: Hookdeck API HTTP 502 (transient); retrying CLI command [%s] (attempt %d/%d, next retry after %v)\nstderr excerpt:\n%s\nstdout excerpt:\n%s", + commandSummary, attempt, acceptance502MaxAttempts, acceptance502RetryDelay, + excerptFor502Log(lastStderr, acceptance502LogExcerpt), + excerptFor502Log(lastStdout, acceptance502LogExcerpt)) + time.Sleep(acceptance502RetryDelay) + } + } + if lastErr != nil && combinedOutputLooksLikeHTTP502(lastStdout, lastStderr) { + r.t.Logf("acceptance: Hookdeck API HTTP 502 still failing after %d attempts (command [%s]); giving up. stderr excerpt:\n%s\nstdout excerpt:\n%s", + acceptance502MaxAttempts, commandSummary, + excerptFor502Log(lastStderr, acceptance502LogExcerpt), + excerptFor502Log(lastStdout, acceptance502LogExcerpt)) + } + return lastStdout, lastStderr, lastErr +} + +// RecordedRequest holds a single HTTP request as captured by the recording proxy. +type RecordedRequest struct { + Method string + Path string + Telemetry string +} + +// RecordingProxy is an HTTP server that forwards requests to the real API and +// records each request (including X-Hookdeck-CLI-Telemetry). Use it to run the +// CLI with --api-base pointing at the proxy so all API traffic is captured +// while still hitting the real backend. +type RecordingProxy struct { + t *testing.T + server *httptest.Server + upstream string + mu sync.Mutex + recorded []RecordedRequest +} + +// URL returns the proxy base URL (e.g. http://127.0.0.1:port). Pass this to +// the CLI as --api-base so requests go through the proxy. +func (p *RecordingProxy) URL() string { + return p.server.URL +} + +// Recorded returns a copy of the slice of recorded requests. Safe to call +// after the CLI command has finished. +func (p *RecordingProxy) Recorded() []RecordedRequest { + p.mu.Lock() + defer p.mu.Unlock() + out := make([]RecordedRequest, len(p.recorded)) + copy(out, p.recorded) + return out +} + +// Close shuts down the proxy server. +func (p *RecordingProxy) Close() { + p.server.Close() +} + +// StartRecordingProxy starts an httptest.Server that acts as a reverse proxy to +// upstreamBase (e.g. https://api.hookdeck.com). Every request is recorded +// (method, path, X-Hookdeck-CLI-Telemetry) and then forwarded to the upstream; +// the upstream response is returned to the client. Use with CLIRunner.Run("--api-base", proxy.URL(), "gateway", ...). +func StartRecordingProxy(t *testing.T, upstreamBase string) *RecordingProxy { + t.Helper() + upstream, err := url.Parse(strings.TrimSuffix(upstreamBase, "/")) + require.NoError(t, err, "parse upstream URL") + + p := &RecordingProxy{ + t: t, + upstream: upstream.String(), + recorded: make([]RecordedRequest, 0), + } + + p.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Record before forwarding + p.mu.Lock() + p.recorded = append(p.recorded, RecordedRequest{ + Method: r.Method, + Path: r.URL.Path, + Telemetry: r.Header.Get("X-Hookdeck-CLI-Telemetry"), + }) + p.mu.Unlock() + + // Build upstream request: same method, path, query, body + targetPath := r.URL.Path + if r.URL.RawQuery != "" { + targetPath += "?" + r.URL.RawQuery + } + dest, err := url.Parse(upstream.String() + targetPath) + require.NoError(p.t, err) + + var bodyReader io.Reader + if r.Body != nil { + bodyReader = r.Body + } + + req, err := http.NewRequest(r.Method, dest.String(), bodyReader) + require.NoError(p.t, err) + + // Copy headers that the API cares about + for _, k := range []string{ + "Authorization", "Content-Type", "X-Team-ID", "X-Project-ID", + "X-Hookdeck-CLI-Telemetry", "X-Hookdeck-Client-User-Agent", "User-Agent", + } { + if v := r.Header.Get(k); v != "" { + req.Header.Set(k, v) + } + } + + resp, err := http.DefaultClient.Do(req) + require.NoError(p.t, err) + defer resp.Body.Close() + + // Copy response back + for k, v := range resp.Header { + for _, vv := range v { + w.Header().Add(k, vv) + } + } + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) + })) + + return p +} + +// telemetryPayload is the structure of the X-Hookdeck-CLI-Telemetry header (JSON). +type telemetryPayload struct { + CommandPath string `json:"command_path"` + InvocationID string `json:"invocation_id"` +} + +// AssertTelemetryConsistent checks that every recorded request that has a +// telemetry header shares the same invocation_id and command_path, and that +// command_path equals expectedCommandPath. +func AssertTelemetryConsistent(t *testing.T, recorded []RecordedRequest, expectedCommandPath string) { + t.Helper() + var invocationID, commandPath string + for i, r := range recorded { + if r.Telemetry == "" { + continue + } + var p telemetryPayload + require.NoError(t, json.Unmarshal([]byte(r.Telemetry), &p), "request %d: invalid telemetry JSON: %s", i, r.Telemetry) + if invocationID == "" { + invocationID = p.InvocationID + commandPath = p.CommandPath + } + require.Equal(t, invocationID, p.InvocationID, "request %d (%s %s): invocation_id should be consistent", i, r.Method, r.Path) + require.Equal(t, commandPath, p.CommandPath, "request %d (%s %s): command_path should be consistent", i, r.Method, r.Path) + } + if invocationID == "" && len(recorded) > 0 { + t.Fatalf("telemetry: %d HTTP request(s) recorded but X-Hookdeck-CLI-Telemetry was empty on every one (unset HOOKDECK_CLI_TELEMETRY_DISABLED / config telemetry_disabled for these tests)", len(recorded)) + } + require.Equal(t, expectedCommandPath, commandPath, "command_path should match expected") + require.NotEmpty(t, invocationID, "at least one request should have invocation_id") +} + func init() { // Attempt to load .env file from test/acceptance/.env for local development // In CI, the environment variable will be set directly @@ -86,6 +300,43 @@ func NewCLIRunner(t *testing.T) *CLIRunner { return runner } +// NewCLIRunnerWithConfigPath creates a CLI runner that uses the given config file path. +// It runs "ci --api-key" so the config is populated (api_key, project_id, project_mode, etc.). +// Use this when a test needs a known config path (e.g. to write a minimal variant for another run). +func NewCLIRunnerWithConfigPath(t *testing.T, configPath string) *CLIRunner { + t.Helper() + apiKey := getAcceptanceAPIKey(t) + require.NotEmpty(t, apiKey, "HOOKDECK_CLI_TESTING_API_KEY must be set") + projectRoot, err := filepath.Abs("../..") + require.NoError(t, err, "Failed to get project root path") + runner := &CLIRunner{ + t: t, + apiKey: apiKey, + projectRoot: projectRoot, + configPath: configPath, + } + stdout, stderr, err := runner.Run("ci", "--api-key", apiKey) + require.NoError(t, err, "Failed to authenticate CLI with config at %s: stdout=%s stderr=%s", configPath, stdout, stderr) + return runner +} + +// NewCLIRunnerWithConfigPathNoCI creates a CLI runner that uses the given config file path, +// without running "ci". Use when the config file is already populated (e.g. a minimal config +// written by the test). The runner will pass HOOKDECK_CONFIG_FILE to all Run() calls. +func NewCLIRunnerWithConfigPathNoCI(t *testing.T, configPath string) *CLIRunner { + t.Helper() + apiKey := getAcceptanceAPIKey(t) + require.NotEmpty(t, apiKey, "HOOKDECK_CLI_TESTING_API_KEY must be set") + projectRoot, err := filepath.Abs("../..") + require.NoError(t, err, "Failed to get project root path") + return &CLIRunner{ + t: t, + apiKey: apiKey, + projectRoot: projectRoot, + configPath: configPath, + } +} + // getAcceptanceAPIKey returns the API key for the current acceptance slice. // When ACCEPTANCE_SLICE=1 and HOOKDECK_CLI_TESTING_API_KEY_2 is set, use it; when ACCEPTANCE_SLICE=2 and HOOKDECK_CLI_TESTING_API_KEY_3 is set, use that; else HOOKDECK_CLI_TESTING_API_KEY. func getAcceptanceAPIKey(t *testing.T) string { @@ -176,26 +427,98 @@ func NewManualCLIRunner(t *testing.T) *CLIRunner { func (r *CLIRunner) Run(args ...string) (stdout, stderr string, err error) { r.t.Helper() - // Use the stored project root path (set during NewCLIRunner) - mainGoPath := filepath.Join(r.projectRoot, "main.go") + summary := commandSummaryFor502Log(args) + return r.runWithHTTP502Retry(summary, func() (string, string, error) { + mainGoPath := filepath.Join(r.projectRoot, "main.go") + cmdArgs := append([]string{"run", mainGoPath}, args...) + cmd := exec.Command("go", cmdArgs...) + cmd.Dir = r.projectRoot + if r.configPath != "" { + cmd.Env = appendEnvOverride(os.Environ(), "HOOKDECK_CONFIG_FILE", r.configPath) + } + var stdoutBuf, stderrBuf bytes.Buffer + cmd.Stdout = &stdoutBuf + cmd.Stderr = &stderrBuf + runErr := cmd.Run() + return stdoutBuf.String(), stderrBuf.String(), runErr + }) +} - cmdArgs := append([]string{"run", mainGoPath}, args...) - cmd := exec.Command("go", cmdArgs...) +// RunWithEnv is like Run but merges extraEnv into the process environment (e.g. for HOOKDECK_CLI_USE_SYSTEM_BINARY). +// When extraEnv["HOOKDECK_CLI_USE_SYSTEM_BINARY"] == "1", runs the installed "hookdeck" binary on PATH instead of "go run main.go" +// (e.g. to run tests against 2.0.0 or another installed version). +func (r *CLIRunner) RunWithEnv(extraEnv map[string]string, args ...string) (stdout, stderr string, err error) { + r.t.Helper() - // Set working directory to project root - cmd.Dir = r.projectRoot + summary := commandSummaryFor502Log(args) + return r.runWithHTTP502Retry(summary, func() (string, string, error) { + env := os.Environ() + if r.configPath != "" { + env = appendEnvOverride(env, "HOOKDECK_CONFIG_FILE", r.configPath) + } + for k, v := range extraEnv { + env = appendEnvOverride(env, k, v) + } + + var cmd *exec.Cmd + if extraEnv != nil && extraEnv["HOOKDECK_CLI_USE_SYSTEM_BINARY"] == "1" { + hookdeckPath, lookErr := exec.LookPath("hookdeck") + if lookErr != nil { + return "", "", fmt.Errorf("HOOKDECK_CLI_USE_SYSTEM_BINARY=1 but hookdeck not on PATH: %w", lookErr) + } + cmd = exec.Command(hookdeckPath, args...) + cmd.Dir = r.projectRoot + } else { + mainGoPath := filepath.Join(r.projectRoot, "main.go") + cmdArgs := append([]string{"run", mainGoPath}, args...) + cmd = exec.Command("go", cmdArgs...) + cmd.Dir = r.projectRoot + } + cmd.Env = env + + var stdoutBuf, stderrBuf bytes.Buffer + cmd.Stdout = &stdoutBuf + cmd.Stderr = &stderrBuf + runErr := cmd.Run() + return stdoutBuf.String(), stderrBuf.String(), runErr + }) +} + +// RunListenWithTimeout starts the CLI with the given args (e.g. "--api-base", proxyURL, +// "listen", port, sourceName, "--output", "compact"), lets it run for runDuration, then +// kills the process. Uses a pre-built binary so we terminate the actual listen process +// (not a "go run" parent). Returns stdout, stderr, and the error from Wait (often +// non-nil because the process was killed). Uses the same project root and config env as Run(). +func (r *CLIRunner) RunListenWithTimeout(args []string, runDuration time.Duration) (stdout, stderr string, err error) { + r.t.Helper() + tmpBinary := filepath.Join(r.projectRoot, "hookdeck-listen-test-"+generateTimestamp()) + defer os.Remove(tmpBinary) + buildCmd := exec.Command("go", "build", "-o", tmpBinary, ".") + buildCmd.Dir = r.projectRoot + if err := buildCmd.Run(); err != nil { + return "", "", fmt.Errorf("build CLI for listen test: %w", err) + } + + cmd := exec.Command(tmpBinary, args...) + cmd.Dir = r.projectRoot if r.configPath != "" { cmd.Env = appendEnvOverride(os.Environ(), "HOOKDECK_CONFIG_FILE", r.configPath) } - var stdoutBuf, stderrBuf bytes.Buffer cmd.Stdout = &stdoutBuf cmd.Stderr = &stderrBuf - - err = cmd.Run() - - return stdoutBuf.String(), stderrBuf.String(), err + if err := cmd.Start(); err != nil { + return "", "", err + } + timer := time.AfterFunc(runDuration, func() { + if cmd.Process != nil { + _ = cmd.Process.Kill() + } + }) + defer timer.Stop() + waitErr := cmd.Wait() + return stdoutBuf.String(), stderrBuf.String(), waitErr } // RunFromCwd executes the CLI from the current working directory. @@ -207,33 +530,30 @@ func (r *CLIRunner) Run(args ...string) (stdout, stderr string, err error) { func (r *CLIRunner) RunFromCwd(args ...string) (stdout, stderr string, err error) { r.t.Helper() - // Build a temporary binary tmpBinary := filepath.Join(r.projectRoot, "hookdeck-test-"+generateTimestamp()) - defer os.Remove(tmpBinary) // Clean up after + defer os.Remove(tmpBinary) - // Build the binary in the project root buildCmd := exec.Command("go", "build", "-o", tmpBinary, ".") buildCmd.Dir = r.projectRoot if err := buildCmd.Run(); err != nil { return "", "", fmt.Errorf("failed to build CLI binary: %w", err) } - // Run the binary from the current working directory - cmd := exec.Command(tmpBinary, args...) - // Don't set cmd.Dir - use current working directory - - if r.configPath != "" { - cmd.Env = appendEnvOverride(os.Environ(), "HOOKDECK_CONFIG_FILE", r.configPath) - } - - var stdoutBuf, stderrBuf bytes.Buffer - cmd.Stdout = &stdoutBuf - cmd.Stderr = &stderrBuf - cmd.Stdin = os.Stdin // Allow interactive input + summary := commandSummaryFor502Log(args) + return r.runWithHTTP502Retry(summary, func() (string, string, error) { + cmd := exec.Command(tmpBinary, args...) + if r.configPath != "" { + cmd.Env = appendEnvOverride(os.Environ(), "HOOKDECK_CONFIG_FILE", r.configPath) + } - err = cmd.Run() + var stdoutBuf, stderrBuf bytes.Buffer + cmd.Stdout = &stdoutBuf + cmd.Stderr = &stderrBuf + cmd.Stdin = os.Stdin - return stdoutBuf.String(), stderrBuf.String(), err + runErr := cmd.Run() + return stdoutBuf.String(), stderrBuf.String(), runErr + }) } // RunExpectSuccess runs the CLI command and fails the test if it returns an error diff --git a/test/acceptance/helpers_502_test.go b/test/acceptance/helpers_502_test.go new file mode 100644 index 0000000..64d8597 --- /dev/null +++ b/test/acceptance/helpers_502_test.go @@ -0,0 +1,54 @@ +//go:build basic + +package acceptance + +import "testing" + +func TestCombinedOutputLooksLikeHTTP502(t *testing.T) { + t.Parallel() + tests := []struct { + name string + stdout string + stderr string + want502 bool + }{ + { + name: "cli client error message", + stderr: "Error: unexpected http status code: 502 bad gateway", + want502: true, + }, + { + name: "logrus status field", + stderr: `level=error msg="request failed" status=502`, + want502: true, + }, + { + name: "error code 502", + stderr: `error code: 502`, + want502: true, + }, + { + name: "503 not retried", + stderr: "unexpected http status code: 503", + want502: false, + }, + { + name: "unrelated 502 substring", + stdout: `{"retry":{"response_status_codes":["500","502"]}}`, + want502: false, + }, + { + name: "empty", + want502: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := combinedOutputLooksLikeHTTP502(tt.stdout, tt.stderr) + if got != tt.want502 { + t.Fatalf("combinedOutputLooksLikeHTTP502(%q, %q) = %v, want %v", tt.stdout, tt.stderr, got, tt.want502) + } + }) + } +} diff --git a/test/acceptance/run_parallel.sh b/test/acceptance/run_parallel.sh index cbe136a..2e4b00c 100755 --- a/test/acceptance/run_parallel.sh +++ b/test/acceptance/run_parallel.sh @@ -1,10 +1,13 @@ #!/usr/bin/env bash -# Run acceptance tests in three parallel slices (same as CI). +# Run acceptance tests in four parallel jobs (same as CI): three matrix slices + telemetry. # Requires HOOKDECK_CLI_TESTING_API_KEY, HOOKDECK_CLI_TESTING_API_KEY_2, and HOOKDECK_CLI_TESTING_API_KEY_3 in environment or test/acceptance/.env. # Run from the repository root. # +# Matrix slices set HOOKDECK_CLI_TELEMETRY_DISABLED=1. The telemetry slice sets it to 0 (matches CI). +# -tags=telemetry (telemetry_test.go and telemetry_listen_test.go). +# # Output: each slice writes to a log file so you can see which run produced what. -# Logs are written to test/acceptance/logs/slice0.log, slice1.log, slice2.log (created on first run). +# Logs are written to test/acceptance/logs/slice0.log, slice1.log, slice2.log, telemetry.log (created on first run). set -e @@ -24,38 +27,51 @@ mkdir -p "$LOG_DIR" SLICE0_LOG="$LOG_DIR/slice0.log" SLICE1_LOG="$LOG_DIR/slice1.log" SLICE2_LOG="$LOG_DIR/slice2.log" +TELEMETRY_LOG="$LOG_DIR/telemetry.log" SLICE0_TAGS="basic connection source destination gateway mcp listen project_use connection_list connection_upsert connection_error_hints connection_oauth_aws connection_update" SLICE1_TAGS="request event" SLICE2_TAGS="attempt metrics issue transformation" run_slice0() { - ACCEPTANCE_SLICE=0 go test -tags="$SLICE0_TAGS" ./test/acceptance/... -v -timeout 12m > "$SLICE0_LOG" 2>&1 + ACCEPTANCE_SLICE=0 HOOKDECK_CLI_TELEMETRY_DISABLED=1 go test -tags="$SLICE0_TAGS" ./test/acceptance/... -v -timeout 12m > "$SLICE0_LOG" 2>&1 } run_slice1() { - ACCEPTANCE_SLICE=1 go test -tags="$SLICE1_TAGS" ./test/acceptance/... -v -timeout 12m > "$SLICE1_LOG" 2>&1 + ACCEPTANCE_SLICE=1 HOOKDECK_CLI_TELEMETRY_DISABLED=1 go test -tags="$SLICE1_TAGS" ./test/acceptance/... -v -timeout 12m > "$SLICE1_LOG" 2>&1 } run_slice2() { - ACCEPTANCE_SLICE=2 go test -tags="$SLICE2_TAGS" ./test/acceptance/... -v -timeout 12m > "$SLICE2_LOG" 2>&1 + ACCEPTANCE_SLICE=2 HOOKDECK_CLI_TELEMETRY_DISABLED=1 go test -tags="$SLICE2_TAGS" ./test/acceptance/... -v -timeout 12m > "$SLICE2_LOG" 2>&1 +} + +run_telemetry() { + ( + export ACCEPTANCE_SLICE=0 + export HOOKDECK_CLI_TELEMETRY_DISABLED=0 + go test -tags=telemetry ./test/acceptance/... -v -timeout 12m + ) > "$TELEMETRY_LOG" 2>&1 } -echo "Running acceptance tests in parallel (slice 0, 1, and 2)..." +echo "Running acceptance tests in parallel (slice 0, 1, 2, and telemetry)..." echo " Slice 0 -> $SLICE0_LOG" echo " Slice 1 -> $SLICE1_LOG" echo " Slice 2 -> $SLICE2_LOG" +echo " Telemetry -> $TELEMETRY_LOG" run_slice0 & PID0=$! run_slice1 & PID1=$! run_slice2 & PID2=$! +run_telemetry & +PIDT=$! FAIL=0 wait $PID0 || FAIL=1 wait $PID1 || FAIL=1 wait $PID2 || FAIL=1 +wait $PIDT || FAIL=1 if [ $FAIL -eq 1 ]; then echo "" @@ -63,8 +79,9 @@ if [ $FAIL -eq 1 ]; then [ ! -f "$SLICE0_LOG" ] || (echo "--- slice 0 ---" && tail -50 "$SLICE0_LOG") [ ! -f "$SLICE1_LOG" ] || (echo "--- slice 1 ---" && tail -50 "$SLICE1_LOG") [ ! -f "$SLICE2_LOG" ] || (echo "--- slice 2 ---" && tail -50 "$SLICE2_LOG") + [ ! -f "$TELEMETRY_LOG" ] || (echo "--- telemetry ---" && tail -50 "$TELEMETRY_LOG") fi echo "" -echo "Logs: $SLICE0_LOG $SLICE1_LOG $SLICE2_LOG" +echo "Logs: $SLICE0_LOG $SLICE1_LOG $SLICE2_LOG $TELEMETRY_LOG" exit $FAIL diff --git a/test/acceptance/source_config_json_test.go b/test/acceptance/source_config_json_test.go index d52acdb..3ecab7e 100644 --- a/test/acceptance/source_config_json_test.go +++ b/test/acceptance/source_config_json_test.go @@ -1,3 +1,5 @@ +//go:build source + package acceptance import ( diff --git a/test/acceptance/telemetry_listen_test.go b/test/acceptance/telemetry_listen_test.go new file mode 100644 index 0000000..1f19053 --- /dev/null +++ b/test/acceptance/telemetry_listen_test.go @@ -0,0 +1,46 @@ +//go:build telemetry + +package acceptance + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestTelemetryListenProxy runs "listen" with the CLI pointed at a recording proxy +// that forwards to the real API. It lets listen run for a few seconds (so it can +// perform initial API calls: get sources, get connections, create session), then +// stops it. Asserts that every API request in that run has the same invocation_id +// and command_path "hookdeck listen". +// +// Build tag telemetry: runs only in the acceptance-telemetry CI job (telemetry enabled), +// not in matrix slices where HOOKDECK_CLI_TELEMETRY_DISABLED=1. +func TestTelemetryListenProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + + cli := NewCLIRunner(t) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + + timestamp := generateTimestamp() + sourceName := "test-telemetry-" + timestamp + + _, _, _ = cli.RunListenWithTimeout([]string{ + "--api-base", proxy.URL(), + "listen", "9999", sourceName, + "--output", "compact", + }, 6*time.Second) + // Process is killed after 6s; we don't assert on exit error + + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1, + "expected at least one API request from listen (sources, connections, or cli-sessions)") + for i, req := range recorded { + t.Logf("listen API request %d: %s %s (telemetry: %s)", i+1, req.Method, req.Path, req.Telemetry) + } + AssertTelemetryConsistent(t, recorded, "hookdeck listen") +} diff --git a/test/acceptance/telemetry_test.go b/test/acceptance/telemetry_test.go new file mode 100644 index 0000000..9e15e4c --- /dev/null +++ b/test/acceptance/telemetry_test.go @@ -0,0 +1,1333 @@ +//go:build telemetry + +package acceptance + +import ( + "bytes" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/BurntSushi/toml" + "github.com/stretchr/testify/require" +) + +// Telemetry proxy tests run commands through a recording proxy and assert that +// every API request from that run has the same invocation_id and command_path. +// +// Rule: one CLI command must produce exactly one command_path (and one +// invocation_id) on every API request. Commands that make multiple API calls +// (e.g. connection upsert does ValidateAPIKey, ListConnections, UpsertConnection) +// must send the same command_path on all of them. AssertTelemetryConsistent +// enforces this; if we had sent whoami / connection list / connection upsert +// for a single "connection upsert" run, that test would fail. +// +// Every test requires the command to succeed (require.NoError). Tests that need +// a real resource create it first without the proxy, then run the command under +// test with the proxy so the proxy only sees that one command. + +// logRecordedTelemetry writes each recorded request's method, path, and telemetry +// (command_path, invocation_id) to the test log. Run with -v to see it. +func logRecordedTelemetry(t *testing.T, recorded []RecordedRequest) { + t.Helper() + t.Logf("DEBUG recorded telemetry: %d request(s)", len(recorded)) + for i, r := range recorded { + t.Logf(" [%d] %s %s", i+1, r.Method, r.Path) + if r.Telemetry == "" { + t.Logf(" (no telemetry header)") + continue + } + var p struct { + CommandPath string `json:"command_path"` + InvocationID string `json:"invocation_id"` + } + if err := json.Unmarshal([]byte(r.Telemetry), &p); err != nil { + t.Logf(" telemetry raw: %s", r.Telemetry) + continue + } + t.Logf(" command_path=%q invocation_id=%q", p.CommandPath, p.InvocationID) + } +} + +// TestTelemetryLoginProxy verifies what we send when we run "hookdeck login --api-key": +// exactly one API call (GET /2025-07-01/cli-auth/validate) with one command_path and one +// invocation_id. Uses the same proxy approach as other telemetry tests (record then forward +// to the real API). Requires HOOKDECK_CLI_TESTING_CLI_KEY (the validate endpoint accepts +// CLI keys from interactive login; API/CI keys may return 401). +func TestTelemetryLoginProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cliKey := os.Getenv("HOOKDECK_CLI_TESTING_CLI_KEY") + if cliKey == "" { + t.Skip("Skipping login telemetry test: HOOKDECK_CLI_TESTING_CLI_KEY must be set (validate endpoint accepts CLI key)") + } + cli := NewCLIRunner(t) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + stdout, stderr, err := cli.Run("--api-base", proxy.URL(), "login", "--api-key", cliKey) + require.NoError(t, err, "login must succeed (proxy forwards to real API); stdout=%q stderr=%q", stdout, stderr) + recorded := proxy.Recorded() + require.Len(t, recorded, 1, "login with --api-key should make exactly one API call (ValidateAPIKey); got %d", len(recorded)) + AssertTelemetryConsistent(t, recorded, "hookdeck login") +} + +func TestTelemetryGatewayConnectionListProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "connection", "list") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway connection list") +} + +func TestTelemetryGatewayConnectionGetProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID := createTestConnection(t, cli) + defer deleteConnection(t, cli, connID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "connection", "get", connID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway connection get") +} + +func TestTelemetryGatewayConnectionUpsertProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + ts := generateTimestamp() + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + var conn Connection + require.NoError(t, cli.RunJSON(&conn, + "--api-base", proxy.URL(), + "gateway", "connection", "upsert", "telemetry-upsert-"+ts, + "--source-name", "telemetry-us-"+ts, "--source-type", "WEBHOOK", + "--destination-name", "telemetry-ud-"+ts, "--destination-type", "CLI", "--destination-cli-path", "/")) + defer deleteConnection(t, cli, conn.ID) + require.NotEmpty(t, conn.ID) + recorded := proxy.Recorded() + // Debug: log recorded telemetry so we can see how many API calls and what command_path each sends. + logRecordedTelemetry(t, recorded) + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway connection upsert") +} + +// TestTelemetryInvocationIDConsistentWhenValidateAPIKeyInPreRun reproduces the v2.0.0 bug: +// when gateway's PersistentPreRunE calls requireGatewayProject() it can perform ValidateAPIKey +// (first API request); then connection's PersistentPreRun runs and overwrites the telemetry +// singleton, so the next requests get a different invocation_id and command_path. This test runs +// "gateway connection upsert" through the recording proxy and asserts every request has the same +// invocation_id and command_path. When the bug occurs (3+ requests with ValidateAPIKey in +// PreRun), v2.0.0 sends inconsistent telemetry and AssertTelemetryConsistent fails; with the root +// fix (set invocation ID only when empty) all requests share one ID and the test passes. +func TestTelemetryInvocationIDConsistentWhenValidateAPIKeyInPreRun(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + tmpDir := t.TempDir() + fullConfigPath := filepath.Join(tmpDir, "full.toml") + minimalConfigPath := filepath.Join(tmpDir, "minimal.toml") + + runnerFull := NewCLIRunnerWithConfigPath(t, fullConfigPath) + ts := generateTimestamp() + connName := "telemetry-inv-id-" + ts + var conn Connection + require.NoError(t, runnerFull.RunJSON(&conn, + "gateway", "connection", "create", + "--source-name", "telemetry-inv-src-"+ts, "--source-type", "WEBHOOK", + "--destination-name", "telemetry-inv-dst-"+ts, "--destination-type", "CLI", "--destination-cli-path", "/", + "--name", connName)) + defer deleteConnection(t, runnerFull, conn.ID) + require.NotEmpty(t, conn.ID) + + // Minimal config (api_key + project_id only) so requireGatewayProject() would call ValidateAPIKey + // if project_type is not set. Running from tmpDir with this config aims to trigger 3 requests. + var fullStruct struct { + Default struct { + APIKey string `toml:"api_key"` + ProjectID string `toml:"project_id"` + } `toml:"default"` + } + data, err := os.ReadFile(fullConfigPath) + require.NoError(t, err, "read full config") + require.NoError(t, toml.Unmarshal(data, &fullStruct), "parse full config") + require.NotEmpty(t, fullStruct.Default.APIKey) + require.NotEmpty(t, fullStruct.Default.ProjectID) + minimalStruct := struct { + Default struct { + APIKey string `toml:"api_key"` + ProjectID string `toml:"project_id"` + } `toml:"default"` + }{Default: fullStruct.Default} + minimalData, err := toml.Marshal(minimalStruct) + require.NoError(t, err) + require.NoError(t, os.WriteFile(minimalConfigPath, minimalData, 0600)) + + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + + // Run CLI binary from tmpDir with minimal config so only our file is used. + projectRoot, err := filepath.Abs("../..") + require.NoError(t, err) + cliBinary := filepath.Join(tmpDir, "hookdeck-test-binary") + buildCmd := exec.Command("go", "build", "-o", cliBinary, ".") + buildCmd.Dir = projectRoot + require.NoError(t, buildCmd.Run(), "build CLI for test") + runCmd := exec.Command(cliBinary, + "--api-base", proxy.URL(), + "--hookdeck-config", "minimal.toml", + "gateway", "connection", "upsert", connName) + runCmd.Dir = tmpDir + runCmd.Env = appendEnvOverride(os.Environ(), "HOOKDECK_CONFIG_FILE", filepath.Join(tmpDir, "minimal.toml")) + var stdoutBuf, stderrBuf bytes.Buffer + runCmd.Stdout = &stdoutBuf + runCmd.Stderr = &stderrBuf + err = runCmd.Run() + require.NoError(t, err, "connection upsert must succeed; stdout=%q stderr=%q", stdoutBuf.String(), stderrBuf.String()) + + recorded := proxy.Recorded() + logRecordedTelemetry(t, recorded) + require.GreaterOrEqual(t, len(recorded), 2, "expected at least ListConnections + UpsertConnection; got %d", len(recorded)) + // When 3 requests occur (ValidateAPIKey in gateway PreRun + List + Upsert), v2.0.0 sends + // different invocation_ids on request 1 vs 2–3; this assertion fails and demonstrates the bug. + AssertTelemetryConsistent(t, recorded, "hookdeck gateway connection upsert") +} + +// TestTelemetryGatewayConnectionUpsertThreeRequestsProxy verifies that when connection upsert +// makes multiple API calls (ListConnections + UpsertConnection, and optionally ValidateAPIKey), +// every request has the same command_path and invocation_id. This reproduces the scenario that +// previously produced three different command_paths in logging. +// +// Combination that yields multiple requests: run upsert with only the connection name +// (no --source-* / --destination-*) so the CLI calls ListConnections then UpsertConnection. +// When the config has no project_type (e.g. minimal config with only api_key and project_id), +// the CLI also calls ValidateAPIKey first, yielding three requests total. With project_type +// already set (e.g. from "ci" or default config), only two requests are made. +func TestTelemetryGatewayConnectionUpsertThreeRequestsProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + tmpDir := t.TempDir() + fullConfigPath := filepath.Join(tmpDir, "full.toml") + minimalConfigPath := filepath.Join(tmpDir, "minimal.toml") + + // Populate full config via "ci" so we have api_key, project_id, project_mode, project_type. + runnerFull := NewCLIRunnerWithConfigPath(t, fullConfigPath) + ts := generateTimestamp() + // Create a connection without proxy so it exists; we'll upsert it by name through the proxy. + connName := "telemetry-three-" + ts + var conn Connection + require.NoError(t, runnerFull.RunJSON(&conn, + "gateway", "connection", "create", + "--source-name", "telemetry-three-src-"+ts, "--source-type", "WEBHOOK", + "--destination-name", "telemetry-three-dst-"+ts, "--destination-type", "CLI", "--destination-cli-path", "/", + "--name", connName)) + defer deleteConnection(t, runnerFull, conn.ID) + require.NotEmpty(t, conn.ID) + + // Read full config and write minimal (api_key + project_id only) so the CLI will call ValidateAPIKey. + var fullStruct struct { + Default struct { + APIKey string `toml:"api_key"` + ProjectID string `toml:"project_id"` + } `toml:"default"` + } + data, err := os.ReadFile(fullConfigPath) + require.NoError(t, err, "read full config") + require.NoError(t, toml.Unmarshal(data, &fullStruct), "parse full config") + require.NotEmpty(t, fullStruct.Default.APIKey, "full config must have api_key") + require.NotEmpty(t, fullStruct.Default.ProjectID, "full config must have project_id") + + minimalStruct := struct { + Default struct { + APIKey string `toml:"api_key"` + ProjectID string `toml:"project_id"` + } `toml:"default"` + }{ + Default: fullStruct.Default, + } + minimalData, err := toml.Marshal(minimalStruct) + require.NoError(t, err) + require.NoError(t, os.WriteFile(minimalConfigPath, minimalData, 0600)) + + // Run upsert through the proxy with minimal config and only the connection name (no source/dest flags). + // Pass --hookdeck-config explicitly so the CLI uses only our minimal file (no merge with cwd .hookdeck). + runnerMinimal := NewCLIRunnerWithConfigPathNoCI(t, minimalConfigPath) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err = runnerMinimal.Run("--api-base", proxy.URL(), "--hookdeck-config", minimalConfigPath, "gateway", "connection", "upsert", connName) + require.NoError(t, err, "connection upsert with minimal config and name-only must succeed") + + recorded := proxy.Recorded() + logRecordedTelemetry(t, recorded) + require.GreaterOrEqual(t, len(recorded), 2, + "expected at least two API calls (ListConnections, UpsertConnection); got %d", len(recorded)) + require.LessOrEqual(t, len(recorded), 3, + "expected at most three API calls (ValidateAPIKey, ListConnections, UpsertConnection); got %d", len(recorded)) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway connection upsert") +} + +// TestTelemetryGatewayConnectionUpsertSystemBinary runs "gateway connection upsert" through the +// recording proxy using the installed "hookdeck" binary on PATH (e.g. 2.0.0). With minimal config +// (no project_type) that run triggers ValidateAPIKey, ListConnections, and UpsertConnection — the +// bug in 2.0.0 was that those three requests were sent with three different command_paths the +// backend showed as "Who am I?", "Connection list", and "Connection upsert". +// +// Run with: HOOKDECK_CLI_USE_SYSTEM_BINARY=1 go test -v -run TestTelemetryGatewayConnectionUpsertSystemBinary ./test/acceptance/ -tags=connection +// Ensure the hookdeck on your PATH is 2.0.0 (or the version you want to test). The test skips if +// HOOKDECK_CLI_USE_SYSTEM_BINARY is not set. +func TestTelemetryGatewayConnectionUpsertSystemBinary(t *testing.T) { + if os.Getenv("HOOKDECK_CLI_USE_SYSTEM_BINARY") != "1" { + t.Skip("Skipping unless HOOKDECK_CLI_USE_SYSTEM_BINARY=1 (run against installed hookdeck binary, e.g. 2.0.0)") + } + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + tmpDir := t.TempDir() + fullConfigPath := filepath.Join(tmpDir, "full.toml") + minimalConfigPath := filepath.Join(tmpDir, "minimal.toml") + + // Create connection using current code (go run) so it exists; upsert will be run with system binary. + runnerFull := NewCLIRunnerWithConfigPath(t, fullConfigPath) + ts := generateTimestamp() + connName := "telemetry-sysbin-" + ts + var conn Connection + require.NoError(t, runnerFull.RunJSON(&conn, + "gateway", "connection", "create", + "--source-name", "telemetry-sysbin-src-"+ts, "--source-type", "WEBHOOK", + "--destination-name", "telemetry-sysbin-dst-"+ts, "--destination-type", "CLI", "--destination-cli-path", "/", + "--name", connName)) + defer deleteConnection(t, runnerFull, conn.ID) + require.NotEmpty(t, conn.ID) + + var fullStruct struct { + Default struct { + APIKey string `toml:"api_key"` + ProjectID string `toml:"project_id"` + } `toml:"default"` + } + data, err := os.ReadFile(fullConfigPath) + require.NoError(t, err) + require.NoError(t, toml.Unmarshal(data, &fullStruct)) + require.NotEmpty(t, fullStruct.Default.APIKey) + require.NotEmpty(t, fullStruct.Default.ProjectID) + + minimalStruct := struct { + Default struct { + APIKey string `toml:"api_key"` + ProjectID string `toml:"project_id"` + } `toml:"default"` + }{Default: fullStruct.Default} + minimalData, err := toml.Marshal(minimalStruct) + require.NoError(t, err) + require.NoError(t, os.WriteFile(minimalConfigPath, minimalData, 0600)) + + // Cobra calls OnInitialize (InitConfig) before parsing flags, so Config.ConfigFileFlag is still "" + // at config load time. We must set HOOKDECK_CONFIG_FILE so the minimal config is used; --hookdeck-config + // alone would only apply after parsing, too late for InitConfig. + systemBinary := map[string]string{"HOOKDECK_CLI_USE_SYSTEM_BINARY": "1"} + runnerMinimal := NewCLIRunnerWithConfigPathNoCI(t, minimalConfigPath) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, stderr, err := runnerMinimal.RunWithEnv(systemBinary, + "--api-base", proxy.URL(), "--hookdeck-config", minimalConfigPath, + "gateway", "connection", "upsert", connName) + require.NoError(t, err, "system binary connection upsert failed; stderr=%s", stderr) + + recorded := proxy.Recorded() + // With minimal config (no project_type) we expect 3 API calls: (1) whoami/ValidateAPIKey, (2) ListConnections, (3) UpsertConnection. + // If the binary already has project_type from another source (or is a fixed build), we may see only 2 (list + upsert). + require.GreaterOrEqual(t, len(recorded), 2, "expected at least 2 API calls (list + upsert); got %d", len(recorded)) + require.LessOrEqual(t, len(recorded), 3, "expected at most 3 API calls; got %d", len(recorded)) + + logRecordedTelemetry(t, recorded) + + // Produce the problem: fail if the system binary sent multiple different command_paths in one run + // (the bug the backend showed as "Who am I?", "Connection list", "Connection upsert"). + paths := make([]string, len(recorded)) + for i, r := range recorded { + if r.Telemetry == "" { + continue + } + var p struct { + CommandPath string `json:"command_path"` + } + if err := json.Unmarshal([]byte(r.Telemetry), &p); err != nil { + continue + } + paths[i] = p.CommandPath + } + uniquePaths := make(map[string]struct{}) + for _, cp := range paths { + if cp != "" { + uniquePaths[cp] = struct{}{} + } + } + if len(uniquePaths) > 1 { + t.Fatalf("BUG REPRODUCED: system binary sent %d different command_paths in one run (expected one). Paths: %v. "+ + "Backend would show these as separate commands (e.g. Who am I?, Connection list, Connection upsert). "+ + "Use the fixed CLI (go build) so all requests share the same command_path.", len(uniquePaths), paths) + } +} + +func TestTelemetryGatewaySourceListProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "source", "list") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway source list") +} + +func TestTelemetryGatewayDestinationListProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "destination", "list") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway destination list") +} + +func TestTelemetryWhoamiProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "whoami") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck whoami") +} + +func TestTelemetryGatewayMetricsEventsProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + runTelemetryProxyTestSuccess(t, + []string{"gateway", "metrics", "events", "--start", "2025-01-01T00:00:00Z", "--end", "2025-01-02T00:00:00Z", "--measures", "count"}, + "hookdeck gateway metrics events") +} + +func TestTelemetryGatewayIssueListProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "issue", "list") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway issue list") +} + +func TestTelemetryGatewayTransformationListProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "transformation", "list") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway transformation list") +} + +func TestTelemetryGatewayEventListProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "event", "list") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway event list") +} + +func TestTelemetryGatewayRequestListProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "request", "list") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway request list") +} + +// runTelemetryProxyTestSuccess runs the CLI through the proxy, requires success, +// and asserts all recorded requests have consistent telemetry for the expected command. +// Use when the command can succeed with the given args (e.g. list, create with valid payload). +func runTelemetryProxyTestSuccess(t *testing.T, args []string, expectedCommandPath string) { + t.Helper() + cli := NewCLIRunner(t) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + fullArgs := append([]string{"--api-base", proxy.URL()}, args...) + _, _, err := cli.Run(fullArgs...) + require.NoError(t, err, "command must succeed so all recorded requests are from this single invocation") + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1, "expected at least one API request for %q", expectedCommandPath) + AssertTelemetryConsistent(t, recorded, expectedCommandPath) +} + +// --- Connection (remaining) --- +func TestTelemetryGatewayConnectionCreateProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + ts := generateTimestamp() + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + var conn Connection + require.NoError(t, cli.RunJSON(&conn, + "--api-base", proxy.URL(), + "gateway", "connection", "create", + "--name", "telemetry-conn-"+ts, "--source-name", "s-"+ts, "--source-type", "WEBHOOK", + "--destination-name", "d-"+ts, "--destination-type", "CLI", "--destination-cli-path", "/")) + defer deleteConnection(t, cli, conn.ID) + require.NotEmpty(t, conn.ID) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway connection create") +} +func TestTelemetryGatewayConnectionUpdateProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID := createTestConnection(t, cli) + defer deleteConnection(t, cli, connID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "connection", "update", connID, "--description", "telemetry-desc") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway connection update") +} +func TestTelemetryGatewayConnectionDeleteProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID := createTestConnection(t, cli) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "connection", "delete", connID, "--force") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway connection delete") +} +func TestTelemetryGatewayConnectionEnableProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID := createTestConnection(t, cli) + defer deleteConnection(t, cli, connID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "connection", "enable", connID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway connection enable") +} +func TestTelemetryGatewayConnectionDisableProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID := createTestConnection(t, cli) + defer deleteConnection(t, cli, connID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "connection", "disable", connID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway connection disable") +} +func TestTelemetryGatewayConnectionPauseProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID := createTestConnection(t, cli) + defer deleteConnection(t, cli, connID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "connection", "pause", connID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway connection pause") +} +func TestTelemetryGatewayConnectionUnpauseProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID := createTestConnection(t, cli) + defer deleteConnection(t, cli, connID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "connection", "unpause", connID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway connection unpause") +} + +// --- Source (remaining) --- +func TestTelemetryGatewaySourceGetProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + srcID := createTestSource(t, cli) + defer deleteSource(t, cli, srcID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "source", "get", srcID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway source get") +} +func TestTelemetryGatewaySourceCreateProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + var src Source + require.NoError(t, cli.RunJSON(&src, + "--api-base", proxy.URL(), + "gateway", "source", "create", "--name", "telemetry-src-"+generateTimestamp(), "--type", "WEBHOOK")) + defer deleteSource(t, cli, src.ID) + require.NotEmpty(t, src.ID) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway source create") +} +func TestTelemetryGatewaySourceUpdateProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + srcID := createTestSource(t, cli) + defer deleteSource(t, cli, srcID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "source", "update", srcID, "--name", "telemetry-src-updated") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway source update") +} +func TestTelemetryGatewaySourceDeleteProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + srcID := createTestSource(t, cli) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "source", "delete", srcID, "--force") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway source delete") +} +func TestTelemetryGatewaySourceEnableProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + srcID := createTestSource(t, cli) + defer deleteSource(t, cli, srcID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "source", "enable", srcID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway source enable") +} +func TestTelemetryGatewaySourceDisableProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + srcID := createTestSource(t, cli) + defer deleteSource(t, cli, srcID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "source", "disable", srcID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway source disable") +} +func TestTelemetryGatewaySourceUpsertProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + runTelemetryProxyTestSuccess(t, + []string{"gateway", "source", "upsert", "telemetry-src-upsert-"+generateTimestamp(), "--type", "WEBHOOK"}, + "hookdeck gateway source upsert") +} +func TestTelemetryGatewaySourceCountProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "source", "count") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway source count") +} + +// --- Destination (remaining) --- +func TestTelemetryGatewayDestinationGetProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + dstID := createTestDestination(t, cli) + defer deleteDestination(t, cli, dstID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "destination", "get", dstID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway destination get") +} +func TestTelemetryGatewayDestinationCreateProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + var dst Destination + require.NoError(t, cli.RunJSON(&dst, + "--api-base", proxy.URL(), + "gateway", "destination", "create", "--name", "telemetry-dst-"+generateTimestamp(), "--type", "HTTP", "--url", "https://example.com")) + defer deleteDestination(t, cli, dst.ID) + require.NotEmpty(t, dst.ID) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway destination create") +} +func TestTelemetryGatewayDestinationUpdateProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + dstID := createTestDestination(t, cli) + defer deleteDestination(t, cli, dstID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "destination", "update", dstID, "--name", "telemetry-dst-updated") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway destination update") +} +func TestTelemetryGatewayDestinationDeleteProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + dstID := createTestDestination(t, cli) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "destination", "delete", dstID, "--force") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway destination delete") +} +func TestTelemetryGatewayDestinationEnableProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + dstID := createTestDestination(t, cli) + defer deleteDestination(t, cli, dstID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "destination", "enable", dstID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway destination enable") +} +func TestTelemetryGatewayDestinationDisableProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + dstID := createTestDestination(t, cli) + defer deleteDestination(t, cli, dstID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "destination", "disable", dstID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway destination disable") +} +func TestTelemetryGatewayDestinationUpsertProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + runTelemetryProxyTestSuccess(t, + []string{"gateway", "destination", "upsert", "telemetry-dst-upsert-"+generateTimestamp(), "--type", "HTTP", "--url", "https://example.com"}, + "hookdeck gateway destination upsert") +} +func TestTelemetryGatewayDestinationCountProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "destination", "count") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway destination count") +} + +// --- Transformation (remaining) --- +func TestTelemetryGatewayTransformationGetProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + trnID := createTestTransformation(t, cli) + defer deleteTransformation(t, cli, trnID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "transformation", "get", trnID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway transformation get") +} +func TestTelemetryGatewayTransformationCreateProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + minCode := `addHandler("transform", (request, context) => { return request; });` + var trn Transformation + require.NoError(t, cli.RunJSON(&trn, + "--api-base", proxy.URL(), + "gateway", "transformation", "create", "--name", "telemetry-trn-"+generateTimestamp(), "--code", minCode)) + defer deleteTransformation(t, cli, trn.ID) + require.NotEmpty(t, trn.ID) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway transformation create") +} +func TestTelemetryGatewayTransformationUpdateProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + trnID := createTestTransformation(t, cli) + defer deleteTransformation(t, cli, trnID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "transformation", "update", trnID, "--name", "telemetry-trn-updated") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway transformation update") +} +func TestTelemetryGatewayTransformationDeleteProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + trnID := createTestTransformation(t, cli) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "transformation", "delete", trnID, "--force") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway transformation delete") +} +func TestTelemetryGatewayTransformationRunProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + runTelemetryProxyTestSuccess(t, + []string{"gateway", "transformation", "run", "--code", `addHandler("transform", (request, context) => { return request; });`, "--request", `{"headers":{}}`}, + "hookdeck gateway transformation run") +} +func TestTelemetryGatewayTransformationUpsertProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + runTelemetryProxyTestSuccess(t, + []string{"gateway", "transformation", "upsert", "telemetry-trn-upsert-"+generateTimestamp(), "--code", `addHandler("transform", (request, context) => { return request; });`}, + "hookdeck gateway transformation upsert") +} +func TestTelemetryGatewayTransformationCountProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "transformation", "count") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway transformation count") +} +func TestTelemetryGatewayTransformationExecutionsListProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + trnID := createTestTransformation(t, cli) + defer deleteTransformation(t, cli, trnID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "transformation", "executions", "list", trnID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway transformation executions list") +} +func TestTelemetryGatewayTransformationExecutionsGetProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + trnID := createTestTransformation(t, cli) + defer deleteTransformation(t, cli, trnID) + // Run transformation to create an execution + _, _, err := cli.Run("gateway", "transformation", "run", "--id", trnID, "--request", `{"headers":{}}`) + require.NoError(t, err) + var listResp struct { + Models []struct { + ID string `json:"id"` + } `json:"models"` + } + require.NoError(t, cli.RunJSON(&listResp, "gateway", "transformation", "executions", "list", trnID)) + if len(listResp.Models) == 0 { + t.Skip("no executions from run; skipping executions get telemetry test") + return + } + execID := listResp.Models[0].ID + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err = cli.Run("--api-base", proxy.URL(), "gateway", "transformation", "executions", "get", trnID, execID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway transformation executions get") +} + +// --- Event (remaining) --- +func TestTelemetryGatewayEventGetProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID, eventID := createConnectionAndTriggerEvent(t, cli) + defer deleteConnection(t, cli, connID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "event", "get", eventID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway event get") +} +func TestTelemetryGatewayEventRetryProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID, eventID := createConnectionAndTriggerEvent(t, cli) + defer deleteConnection(t, cli, connID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "event", "retry", eventID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway event retry") +} +func TestTelemetryGatewayEventCancelProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID, eventID := createConnectionAndTriggerEvent(t, cli) + defer deleteConnection(t, cli, connID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "event", "cancel", eventID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway event cancel") +} +func TestTelemetryGatewayEventMuteProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID, eventID := createConnectionAndTriggerEvent(t, cli) + defer deleteConnection(t, cli, connID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "event", "mute", eventID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway event mute") +} +func TestTelemetryGatewayEventRawBodyProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID, eventID := createConnectionAndTriggerEvent(t, cli) + defer deleteConnection(t, cli, connID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "event", "raw-body", eventID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway event raw-body") +} + +// --- Request (remaining) --- +func TestTelemetryGatewayRequestGetProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID, _ := createConnectionAndTriggerEvent(t, cli) + defer deleteConnection(t, cli, connID) + var conn Connection + require.NoError(t, cli.RunJSON(&conn, "gateway", "connection", "get", connID)) + requests := pollForRequestsBySourceID(t, cli, conn.Source.ID) + require.NotEmpty(t, requests) + requestID := requests[0].ID + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "request", "get", requestID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway request get") +} +func TestTelemetryGatewayRequestRetryProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID, _ := createConnectionAndTriggerEvent(t, cli) + defer deleteConnection(t, cli, connID) + var conn Connection + require.NoError(t, cli.RunJSON(&conn, "gateway", "connection", "get", connID)) + requests := pollForRequestsBySourceID(t, cli, conn.Source.ID) + require.NotEmpty(t, requests) + requestID := requests[0].ID + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, stderr, err := cli.Run("--api-base", proxy.URL(), "gateway", "request", "retry", requestID) + if err != nil { + t.Skipf("Skipping request retry telemetry test: API rejected retry (exit 1). stderr=%q", stderr) + return + } + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway request retry") +} +func TestTelemetryGatewayRequestEventsProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID, _ := createConnectionAndTriggerEvent(t, cli) + defer deleteConnection(t, cli, connID) + var conn Connection + require.NoError(t, cli.RunJSON(&conn, "gateway", "connection", "get", connID)) + requests := pollForRequestsBySourceID(t, cli, conn.Source.ID) + require.NotEmpty(t, requests) + requestID := requests[0].ID + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "request", "events", requestID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway request events") +} +func TestTelemetryGatewayRequestIgnoredEventsProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID, _ := createConnectionAndTriggerEvent(t, cli) + defer deleteConnection(t, cli, connID) + var conn Connection + require.NoError(t, cli.RunJSON(&conn, "gateway", "connection", "get", connID)) + requests := pollForRequestsBySourceID(t, cli, conn.Source.ID) + require.NotEmpty(t, requests) + requestID := requests[0].ID + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "request", "ignored-events", requestID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway request ignored-events") +} +func TestTelemetryGatewayRequestRawBodyProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID, _ := createConnectionAndTriggerEvent(t, cli) + defer deleteConnection(t, cli, connID) + var conn Connection + require.NoError(t, cli.RunJSON(&conn, "gateway", "connection", "get", connID)) + requests := pollForRequestsBySourceID(t, cli, conn.Source.ID) + require.NotEmpty(t, requests) + requestID := requests[0].ID + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "request", "raw-body", requestID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway request raw-body") +} + +// --- Attempt --- +func TestTelemetryGatewayAttemptListProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID, eventID := createConnectionAndTriggerEvent(t, cli) + defer deleteConnection(t, cli, connID) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "attempt", "list", "--event-id", eventID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway attempt list") +} +func TestTelemetryGatewayAttemptGetProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID, eventID := createConnectionAndTriggerEvent(t, cli) + defer deleteConnection(t, cli, connID) + attempts := pollForAttemptsByEventID(t, cli, eventID) + require.NotEmpty(t, attempts) + attemptID := attempts[0].ID + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "attempt", "get", attemptID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway attempt get") +} +func TestTelemetryGatewayAttemptRetryProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + connID, eventID := createConnectionAndTriggerEvent(t, cli) + defer deleteConnection(t, cli, connID) + attempts := pollForAttemptsByEventID(t, cli, eventID) + require.NotEmpty(t, attempts) + attemptID := attempts[0].ID + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, stderr, err := cli.Run("--api-base", proxy.URL(), "gateway", "attempt", "retry", attemptID) + if err != nil { + t.Skipf("Skipping attempt retry telemetry test: API rejected retry (exit 1). stderr=%q", stderr) + return + } + recorded := proxy.Recorded() + if len(recorded) == 0 { + t.Skip("Skipping attempt retry telemetry test: no API request was made (retry may be no-op in this state)") + return + } + AssertTelemetryConsistent(t, recorded, "hookdeck gateway attempt retry") +} + +// --- Metrics (remaining) --- +func TestTelemetryGatewayMetricsRequestsProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + runTelemetryProxyTestSuccess(t, + []string{"gateway", "metrics", "requests", "--start", "2025-01-01T00:00:00Z", "--end", "2025-01-02T00:00:00Z", "--measures", "count"}, + "hookdeck gateway metrics requests") +} +func TestTelemetryGatewayMetricsAttemptsProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + runTelemetryProxyTestSuccess(t, + []string{"gateway", "metrics", "attempts", "--start", "2025-01-01T00:00:00Z", "--end", "2025-01-02T00:00:00Z", "--measures", "count"}, + "hookdeck gateway metrics attempts") +} +func TestTelemetryGatewayMetricsTransformationsProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + runTelemetryProxyTestSuccess(t, + []string{"gateway", "metrics", "transformations", "--start", "2025-01-01T00:00:00Z", "--end", "2025-01-02T00:00:00Z", "--measures", "count"}, + "hookdeck gateway metrics transformations") +} + +// --- Issue (remaining) --- +func TestTelemetryGatewayIssueGetProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + var listResp struct { + Models []Issue `json:"models"` + } + require.NoError(t, cli.RunJSON(&listResp, "gateway", "issue", "list", "--limit", "1")) + if len(listResp.Models) == 0 { + t.Skip("no issues in workspace; skipping issue get telemetry test") + return + } + issueID := listResp.Models[0].ID + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "issue", "get", issueID) + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway issue get") +} +func TestTelemetryGatewayIssueUpdateProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + var listResp struct { + Models []Issue `json:"models"` + } + require.NoError(t, cli.RunJSON(&listResp, "gateway", "issue", "list", "--status", "OPENED", "--limit", "1")) + if len(listResp.Models) == 0 { + t.Skip("no open issues in workspace; skipping issue update telemetry test") + return + } + issueID := listResp.Models[0].ID + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "issue", "update", issueID, "--status", "resolved") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway issue update") +} +func TestTelemetryGatewayIssueDismissProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + var listResp struct { + Models []Issue `json:"models"` + } + require.NoError(t, cli.RunJSON(&listResp, "gateway", "issue", "list", "--status", "OPENED", "--limit", "1")) + if len(listResp.Models) == 0 { + t.Skip("no open issues in workspace; skipping issue dismiss telemetry test") + return + } + issueID := listResp.Models[0].ID + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "issue", "dismiss", issueID, "--force") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway issue dismiss") +} +func TestTelemetryGatewayIssueCountProxy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping acceptance test in short mode") + } + cli := NewCLIRunner(t) + proxy := StartRecordingProxy(t, defaultAPIUpstream) + defer proxy.Close() + _, _, err := cli.Run("--api-base", proxy.URL(), "gateway", "issue", "count") + require.NoError(t, err) + recorded := proxy.Recorded() + require.GreaterOrEqual(t, len(recorded), 1) + AssertTelemetryConsistent(t, recorded, "hookdeck gateway issue count") +}