From c8543e9d0969edc048b1e635192e0b1e6c732453 Mon Sep 17 00:00:00 2001 From: Alfonso Noriega Date: Fri, 13 Mar 2026 11:24:20 +0100 Subject: [PATCH 1/5] feat: track auto-upgrade metrics in Monorail (#22365) Co-Authored-By: Claude Sonnet 4.6 --- .changeset/auto-upgrade-metrics.md | 5 ++ .../src/public/node/hooks/postrun.test.ts | 68 +++++++++++++++++++ .../cli-kit/src/public/node/hooks/postrun.ts | 19 +++++- packages/cli-kit/src/public/node/monorail.ts | 6 ++ 4 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 .changeset/auto-upgrade-metrics.md diff --git a/.changeset/auto-upgrade-metrics.md b/.changeset/auto-upgrade-metrics.md new file mode 100644 index 00000000000..0bbb69d6c21 --- /dev/null +++ b/.changeset/auto-upgrade-metrics.md @@ -0,0 +1,5 @@ +--- +'@shopify/cli-kit': patch +--- + +Track auto-upgrade events (trigger rate, success/failure, package manager, skip reason) in Monorail. diff --git a/packages/cli-kit/src/public/node/hooks/postrun.test.ts b/packages/cli-kit/src/public/node/hooks/postrun.test.ts index 4d2bc6ff6d6..4eb598c09c3 100644 --- a/packages/cli-kit/src/public/node/hooks/postrun.test.ts +++ b/packages/cli-kit/src/public/node/hooks/postrun.test.ts @@ -1,14 +1,21 @@ import {autoUpgradeIfNeeded} from './postrun.js' import {versionToAutoUpgrade, runCLIUpgrade, getOutputUpdateCLIReminder} from '../upgrade.js' import {isMajorVersionChange} from '../version.js' +import {inferPackageManagerForGlobalCLI} from '../is-global.js' +import {addPublicMetadata} from '../metadata.js' import {mockAndCaptureOutput} from '../testing/output.js' import {describe, test, vi, expect, afterEach} from 'vitest' vi.mock('../upgrade.js') vi.mock('../version.js') +vi.mock('../is-global.js') +vi.mock('../metadata.js', () => ({ + addPublicMetadata: vi.fn().mockResolvedValue(undefined), +})) afterEach(() => { mockAndCaptureOutput().clear() + vi.mocked(addPublicMetadata).mockClear() }) describe('autoUpgradeIfNeeded', () => { @@ -35,6 +42,7 @@ describe('autoUpgradeIfNeeded', () => { test('calls runCLIUpgrade for minor/patch upgrade', async () => { vi.mocked(versionToAutoUpgrade).mockReturnValue('3.100.0') vi.mocked(isMajorVersionChange).mockReturnValue(false) + vi.mocked(inferPackageManagerForGlobalCLI).mockReturnValue('npm') vi.mocked(runCLIUpgrade).mockResolvedValue(undefined) await autoUpgradeIfNeeded() @@ -46,6 +54,7 @@ describe('autoUpgradeIfNeeded', () => { const outputMock = mockAndCaptureOutput() vi.mocked(versionToAutoUpgrade).mockReturnValue('3.100.0') vi.mocked(isMajorVersionChange).mockReturnValue(false) + vi.mocked(inferPackageManagerForGlobalCLI).mockReturnValue('npm') vi.mocked(runCLIUpgrade).mockRejectedValue(new Error('upgrade failed')) vi.mocked(getOutputUpdateCLIReminder).mockReturnValue('💡 Version 3.100.0 available! Run `npm install -g @shopify/cli@latest`') @@ -53,4 +62,63 @@ describe('autoUpgradeIfNeeded', () => { expect(outputMock.warn()).toContain('3.100.0') }) + + test('records triggered metric for minor/patch upgrade', async () => { + vi.mocked(versionToAutoUpgrade).mockReturnValue('3.100.0') + vi.mocked(isMajorVersionChange).mockReturnValue(false) + vi.mocked(inferPackageManagerForGlobalCLI).mockReturnValue('npm') + vi.mocked(runCLIUpgrade).mockResolvedValue(undefined) + + await autoUpgradeIfNeeded() + + expect(addPublicMetadata).toHaveBeenCalledWith(expect.any(Function)) + const calls = vi.mocked(addPublicMetadata).mock.calls.map((call) => call[0]()) + expect(calls).toContainEqual( + expect.objectContaining({ + cmd_all_auto_upgrade_triggered: true, + cmd_all_auto_upgrade_package_manager: 'npm', + }), + ) + }) + + test('records triggered metric with major_version skipped_reason for major bump', async () => { + vi.mocked(versionToAutoUpgrade).mockReturnValue('4.0.0') + vi.mocked(isMajorVersionChange).mockReturnValue(true) + vi.mocked(getOutputUpdateCLIReminder).mockReturnValue('upgrade reminder') + + await autoUpgradeIfNeeded() + + const calls = vi.mocked(addPublicMetadata).mock.calls.map((call) => call[0]()) + expect(calls).toContainEqual( + expect.objectContaining({ + cmd_all_auto_upgrade_triggered: true, + cmd_all_auto_upgrade_skipped_reason: 'major_version', + }), + ) + }) + + test('records success=true on successful upgrade', async () => { + vi.mocked(versionToAutoUpgrade).mockReturnValue('3.100.0') + vi.mocked(isMajorVersionChange).mockReturnValue(false) + vi.mocked(inferPackageManagerForGlobalCLI).mockReturnValue('npm') + vi.mocked(runCLIUpgrade).mockResolvedValue(undefined) + + await autoUpgradeIfNeeded() + + const calls = vi.mocked(addPublicMetadata).mock.calls.map((call) => call[0]()) + expect(calls).toContainEqual(expect.objectContaining({cmd_all_auto_upgrade_success: true})) + }) + + test('records success=false on failed upgrade', async () => { + vi.mocked(versionToAutoUpgrade).mockReturnValue('3.100.0') + vi.mocked(isMajorVersionChange).mockReturnValue(false) + vi.mocked(inferPackageManagerForGlobalCLI).mockReturnValue('npm') + vi.mocked(runCLIUpgrade).mockRejectedValue(new Error('upgrade failed')) + vi.mocked(getOutputUpdateCLIReminder).mockReturnValue('upgrade reminder') + + await autoUpgradeIfNeeded() + + const calls = vi.mocked(addPublicMetadata).mock.calls.map((call) => call[0]()) + expect(calls).toContainEqual(expect.objectContaining({cmd_all_auto_upgrade_success: false})) + }) }) diff --git a/packages/cli-kit/src/public/node/hooks/postrun.ts b/packages/cli-kit/src/public/node/hooks/postrun.ts index a7685007935..38d7311ed2c 100644 --- a/packages/cli-kit/src/public/node/hooks/postrun.ts +++ b/packages/cli-kit/src/public/node/hooks/postrun.ts @@ -4,6 +4,7 @@ import {outputDebug, outputWarn} from '../output.js' import {getOutputUpdateCLIReminder, runCLIUpgrade, versionToAutoUpgrade} from '../upgrade.js' import BaseCommand from '../base-command.js' import * as metadata from '../metadata.js' +import {inferPackageManagerForGlobalCLI} from '../is-global.js' import {CLI_KIT_VERSION} from '../../common/version.js' import {isMajorVersionChange} from '../version.js' @@ -40,19 +41,35 @@ export const hook: Hook.Postrun = async ({config, Command}) => { */ export async function autoUpgradeIfNeeded(): Promise { const newerVersion = versionToAutoUpgrade() - if (!newerVersion) return + if (!newerVersion) { + // versionToAutoUpgrade already logged the reason via outputDebug + return + } + if (isMajorVersionChange(CLI_KIT_VERSION, newerVersion)) { outputWarn(getOutputUpdateCLIReminder(newerVersion)) + await metadata.addPublicMetadata(() => ({ + cmd_all_auto_upgrade_triggered: true, + cmd_all_auto_upgrade_skipped_reason: 'major_version', + })) return } + const packageManager = inferPackageManagerForGlobalCLI() + await metadata.addPublicMetadata(() => ({ + cmd_all_auto_upgrade_triggered: true, + cmd_all_auto_upgrade_package_manager: packageManager, + })) + try { await runCLIUpgrade() + await metadata.addPublicMetadata(() => ({cmd_all_auto_upgrade_success: true})) // eslint-disable-next-line no-catch-all/no-catch-all } catch (error) { const errorMessage = `Auto-upgrade failed: ${error}` outputDebug(errorMessage) outputWarn(getOutputUpdateCLIReminder(newerVersion)) + await metadata.addPublicMetadata(() => ({cmd_all_auto_upgrade_success: false})) // Report to Observe as a handled error without showing anything extra to the user const {sendErrorToBugsnag} = await import('../error-handler.js') await sendErrorToBugsnag(new Error(errorMessage), 'expected_error') diff --git a/packages/cli-kit/src/public/node/monorail.ts b/packages/cli-kit/src/public/node/monorail.ts index b687ec9fee4..a14be863b22 100644 --- a/packages/cli-kit/src/public/node/monorail.ts +++ b/packages/cli-kit/src/public/node/monorail.ts @@ -63,6 +63,12 @@ export interface Schemas { cmd_all_timing_prompts_ms?: Optional cmd_all_timing_active_ms?: Optional + // Auto-upgrade + cmd_all_auto_upgrade_triggered?: Optional + cmd_all_auto_upgrade_skipped_reason?: Optional + cmd_all_auto_upgrade_success?: Optional + cmd_all_auto_upgrade_package_manager?: Optional + // Any extension related command cmd_extensions_binary_from_source?: Optional From 64d3105b9e990851b3a1ab4f92ad276c28560102 Mon Sep 17 00:00:00 2001 From: Alfonso Noriega Date: Fri, 13 Mar 2026 12:52:11 +0100 Subject: [PATCH 2/5] fix: use static imports in upgrade.test.ts and fix prettier in postrun.test.ts --- packages/cli-kit/src/public/node/hooks/postrun.test.ts | 4 +++- packages/cli/src/cli/commands/upgrade.test.ts | 9 ++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/cli-kit/src/public/node/hooks/postrun.test.ts b/packages/cli-kit/src/public/node/hooks/postrun.test.ts index 4eb598c09c3..a306d801915 100644 --- a/packages/cli-kit/src/public/node/hooks/postrun.test.ts +++ b/packages/cli-kit/src/public/node/hooks/postrun.test.ts @@ -56,7 +56,9 @@ describe('autoUpgradeIfNeeded', () => { vi.mocked(isMajorVersionChange).mockReturnValue(false) vi.mocked(inferPackageManagerForGlobalCLI).mockReturnValue('npm') vi.mocked(runCLIUpgrade).mockRejectedValue(new Error('upgrade failed')) - vi.mocked(getOutputUpdateCLIReminder).mockReturnValue('💡 Version 3.100.0 available! Run `npm install -g @shopify/cli@latest`') + vi.mocked(getOutputUpdateCLIReminder).mockReturnValue( + '💡 Version 3.100.0 available! Run `npm install -g @shopify/cli@latest`', + ) await autoUpgradeIfNeeded() diff --git a/packages/cli/src/cli/commands/upgrade.test.ts b/packages/cli/src/cli/commands/upgrade.test.ts index 61e04ba2059..947195bb636 100644 --- a/packages/cli/src/cli/commands/upgrade.test.ts +++ b/packages/cli/src/cli/commands/upgrade.test.ts @@ -1,14 +1,13 @@ import Upgrade from './upgrade.js' +import {promptAutoUpgrade, runCLIUpgrade} from '@shopify/cli-kit/node/upgrade' import {describe, test, vi, expect} from 'vitest' -vi.mock('@shopify/cli-kit/node/upgrade', () => ({ - promptAutoUpgrade: vi.fn().mockResolvedValue(true), - runCLIUpgrade: vi.fn().mockResolvedValue(undefined), -})) +vi.mock('@shopify/cli-kit/node/upgrade') describe('upgrade command', () => { test('calls promptAutoUpgrade and runCLIUpgrade', async () => { - const {promptAutoUpgrade, runCLIUpgrade} = await import('@shopify/cli-kit/node/upgrade') + vi.mocked(promptAutoUpgrade).mockResolvedValue(true) + vi.mocked(runCLIUpgrade).mockResolvedValue(undefined) await Upgrade.run([], import.meta.url) From fe0607a8609e95d34a5785a1e222f609d6e99626 Mon Sep 17 00:00:00 2001 From: Alfonso Noriega Date: Fri, 13 Mar 2026 12:58:30 +0100 Subject: [PATCH 3/5] fix: update OCLIF manifest and README for upgraded upgrade command Co-Authored-By: Claude Sonnet 4.6 --- packages/cli/README.md | 6 +++--- packages/cli/oclif.manifest.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/cli/README.md b/packages/cli/README.md index 0ef528b2e4d..07fb8dee74b 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -2765,16 +2765,16 @@ DESCRIPTION ## `shopify upgrade` -Shows details on how to upgrade Shopify CLI. +Upgrades Shopify CLI. ``` USAGE $ shopify upgrade DESCRIPTION - Shows details on how to upgrade Shopify CLI. + Upgrades Shopify CLI. - Shows details on how to upgrade Shopify CLI. + Upgrades Shopify CLI using your package manager. ``` ## `shopify version` diff --git a/packages/cli/oclif.manifest.json b/packages/cli/oclif.manifest.json index 8a053bd1ea3..46d246328f9 100644 --- a/packages/cli/oclif.manifest.json +++ b/packages/cli/oclif.manifest.json @@ -7822,8 +7822,8 @@ ], "args": { }, - "description": "Shows details on how to upgrade Shopify CLI.", - "descriptionWithMarkdown": "Shows details on how to upgrade Shopify CLI.", + "description": "Upgrades Shopify CLI.", + "descriptionWithMarkdown": "Upgrades Shopify CLI using your package manager.", "enableJsonFlag": false, "flags": { }, @@ -7835,7 +7835,7 @@ "pluginName": "@shopify/cli", "pluginType": "core", "strict": true, - "summary": "Shows details on how to upgrade Shopify CLI." + "summary": "Upgrades Shopify CLI." }, "version": { "aliases": [ From e167c7302853276f4653ad8000f1a740cf76d32c Mon Sep 17 00:00:00 2001 From: Alfonso Noriega Date: Fri, 13 Mar 2026 13:11:34 +0100 Subject: [PATCH 4/5] fix: remove unused beforeEach import in is-global.test.ts Co-Authored-By: Claude Sonnet 4.6 --- packages/cli-kit/src/public/node/is-global.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli-kit/src/public/node/is-global.test.ts b/packages/cli-kit/src/public/node/is-global.test.ts index 5142ba9d77b..5ed18d5186c 100644 --- a/packages/cli-kit/src/public/node/is-global.test.ts +++ b/packages/cli-kit/src/public/node/is-global.test.ts @@ -2,7 +2,7 @@ import {currentProcessIsGlobal, inferPackageManagerForGlobalCLI, installGlobalCL import {terminalSupportsPrompting} from './system.js' import {renderSelectPrompt} from './ui.js' import {globalCLIVersion} from './version.js' -import {beforeEach, describe, expect, test, vi, afterEach} from 'vitest' +import {describe, expect, test, vi, afterEach} from 'vitest' vi.mock('./system.js') vi.mock('./ui.js') From 2869478ec64395b54726de073022a6d0db85c043 Mon Sep 17 00:00:00 2001 From: Alfonso Noriega Date: Fri, 13 Mar 2026 13:15:49 +0100 Subject: [PATCH 5/5] fix: correct description field in oclif manifest for upgrade command --- packages/cli/oclif.manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/oclif.manifest.json b/packages/cli/oclif.manifest.json index 46d246328f9..78675c99fcf 100644 --- a/packages/cli/oclif.manifest.json +++ b/packages/cli/oclif.manifest.json @@ -7822,7 +7822,7 @@ ], "args": { }, - "description": "Upgrades Shopify CLI.", + "description": "Upgrades Shopify CLI using your package manager.", "descriptionWithMarkdown": "Upgrades Shopify CLI using your package manager.", "enableJsonFlag": false, "flags": {