diff --git a/.github/workflows/sync-release-to-main.yaml b/.github/workflows/sync-release-to-main.yaml index a61dac8..1777a04 100644 --- a/.github/workflows/sync-release-to-main.yaml +++ b/.github/workflows/sync-release-to-main.yaml @@ -40,15 +40,5 @@ jobs: --base main \ --head release \ --title "chore: sync release to main" \ - --body "Automated sync of release tags back to main." - fi - - - name: Enable auto-merge on sync PR - if: steps.check.outputs.ahead != '0' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - PR=$(gh pr list --repo ${{ github.repository }} --base main --head release --state open --json number --jq '.[0].number') - if [ -n "$PR" ]; then - gh pr merge "$PR" --repo ${{ github.repository }} --merge --auto + --body "Automated sync of release tags back to main. Merge with **merge commit**." fi diff --git a/scripts/generate-registry.ts b/scripts/generate-registry.ts index 6735b8b..4535f37 100644 --- a/scripts/generate-registry.ts +++ b/scripts/generate-registry.ts @@ -9,7 +9,7 @@ * Run: npm run generate:registry */ -import { readFileSync, existsSync, writeFileSync } from 'fs'; +import { existsSync, readFileSync, writeFileSync } from 'fs'; import { join } from 'path'; import * as YAML from 'yaml'; diff --git a/scripts/update-docs.ts b/scripts/update-docs.ts index 2dd03b4..0c3eed4 100644 --- a/scripts/update-docs.ts +++ b/scripts/update-docs.ts @@ -1,7 +1,8 @@ -import { readFileSync, writeFileSync, existsSync } from 'fs'; -import { join, dirname } from 'path'; +import { existsSync, readFileSync, writeFileSync } from 'fs'; +import { dirname, join } from 'path'; import { fileURLToPath } from 'url'; import * as yaml from 'yaml'; + import type { CommandSpec } from '../src/types.js'; const __filename = fileURLToPath(import.meta.url); diff --git a/src/lib/bundle.ts b/src/lib/bundle.ts new file mode 100644 index 0000000..351c14c --- /dev/null +++ b/src/lib/bundle.ts @@ -0,0 +1,156 @@ +import { getStorageConfig } from '@auth/provider.js'; +import { bundle } from '@tigrisdata/storage'; +import { exitWithError } from '@utils/exit.js'; +import { getFormat, getOption, readStdin } from '@utils/options.js'; +import { parseAnyPath } from '@utils/path.js'; +import { createWriteStream, existsSync, readFileSync } from 'fs'; +import { Readable } from 'stream'; +import { pipeline } from 'stream/promises'; + +const MAX_KEYS = 5000; + +function parseKeys(content: string): string[] { + return content + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0 && !line.startsWith('#')); +} + +function detectCompression( + outputPath: string +): 'none' | 'gzip' | 'zstd' | undefined { + if (outputPath.endsWith('.tar.gz') || outputPath.endsWith('.tgz')) { + return 'gzip'; + } + if (outputPath.endsWith('.tar.zst')) { + return 'zstd'; + } + if (outputPath.endsWith('.tar')) { + return 'none'; + } + return undefined; +} + +export default async function bundleCommand(options: Record) { + const bucketArg = getOption(options, ['bucket']); + const keysArg = getOption(options, ['keys', 'k']); + const outputPath = getOption(options, ['output', 'o']); + const compressionArg = getOption(options, ['compression']); + const onError = getOption(options, ['on-error', 'onError'], 'skip'); + const format = getFormat(options); + const jsonMode = format === 'json'; + + // stdout carries binary data when no --output + const stdoutBinary = !outputPath; + + if (!bucketArg) { + exitWithError('Bucket is required'); + } + + const { bucket, path: prefix } = parseAnyPath(bucketArg); + + if (!bucket) { + exitWithError('Invalid bucket'); + } + + // Resolve keys: file, inline, or stdin + let keys: string[]; + + if (keysArg) { + if (keysArg.includes(',')) { + // Commas present → always treat as inline comma-separated keys + keys = keysArg + .split(',') + .map((k) => k.trim()) + .filter((k) => k.length > 0); + } else if (existsSync(keysArg)) { + // No commas and local file exists → read as keys file + keys = parseKeys(readFileSync(keysArg, 'utf-8')); + } else { + // Single key + keys = [keysArg.trim()]; + } + } else if (!process.stdin.isTTY) { + const input = await readStdin(); + keys = parseKeys(input); + } else { + exitWithError('Keys are required. Provide via --keys or pipe to stdin.'); + } + + // Prepend path prefix from bucket arg (e.g. t3://bucket/prefix) + if (prefix) { + const normalizedPrefix = prefix.endsWith('/') ? prefix : `${prefix}/`; + keys = keys.map((key) => `${normalizedPrefix}${key}`); + } + + if (keys.length === 0) { + exitWithError('No keys found'); + } + + if (keys.length > MAX_KEYS) { + exitWithError(`Too many keys (max ${MAX_KEYS}). Got ${keys.length}`); + } + + // Resolve compression: explicit flag > auto-detect from extension > default + let compression: 'none' | 'gzip' | 'zstd' = 'none'; + if (compressionArg) { + compression = compressionArg as 'none' | 'gzip' | 'zstd'; + } else if (outputPath) { + compression = detectCompression(outputPath) ?? 'none'; + } + + if (!stdoutBinary && !jsonMode) { + process.stderr.write(`Bundling ${keys.length} object(s)...\n`); + } + + const config = await getStorageConfig({ withCredentialProvider: true }); + + const { data, error } = await bundle(keys, { + config: { ...config, bucket }, + compression, + onError: onError as 'skip' | 'fail', + }); + + if (error) { + exitWithError(error); + } + + const nodeStream = Readable.fromWeb(data.body as ReadableStream); + + if (outputPath) { + const writeStream = createWriteStream(outputPath); + await pipeline(nodeStream, writeStream); + + if (jsonMode) { + console.log( + JSON.stringify({ + action: 'bundled', + bucket, + keys: keys.length, + compression, + output: outputPath, + }) + ); + } else { + console.log( + `Bundled ${keys.length} object(s) from '${bucket}' to ${outputPath}` + ); + } + } else { + await pipeline(nodeStream, process.stdout); + + if (jsonMode) { + console.error( + JSON.stringify({ + action: 'bundled', + bucket, + keys: keys.length, + compression, + output: 'stdout', + }) + ); + } + } + + process.exit(0); +} diff --git a/src/lib/iam/policies/utils.ts b/src/lib/iam/policies/utils.ts index 0e206b7..961c595 100644 --- a/src/lib/iam/policies/utils.ts +++ b/src/lib/iam/policies/utils.ts @@ -1,12 +1,6 @@ import type { PolicyDocument } from '@tigrisdata/iam'; -export async function readStdin(): Promise { - const chunks: Buffer[] = []; - for await (const chunk of process.stdin) { - chunks.push(chunk); - } - return Buffer.concat(chunks).toString('utf-8'); -} +export { readStdin } from '@utils/options.js'; export function parseDocument(jsonString: string): PolicyDocument { const raw = JSON.parse(jsonString); diff --git a/src/specs.yaml b/src/specs.yaml index 3c8b1ee..01091d7 100644 --- a/src/specs.yaml +++ b/src/specs.yaml @@ -546,6 +546,41 @@ commands: alias: f description: Skip confirmation prompts (alias for --yes) + # bundle + - name: bundle + description: Download multiple objects as a streaming tar archive in a single request. Designed for batch workloads that need many objects without per-object HTTP overhead + examples: + - "tigris bundle my-bucket --keys key1.jpg,key2.jpg --output archive.tar" + - "tigris bundle my-bucket --keys keys.txt --output archive.tar" + - "tigris bundle t3://my-bucket --keys keys.txt --compression gzip -o archive.tar.gz" + - "cat keys.txt | tigris bundle my-bucket > archive.tar" + messages: + onStart: '' + onSuccess: '' + onFailure: 'Bundle failed. Verify the bucket exists and credentials have read access' + arguments: + - name: bucket + required: true + type: positional + description: Bucket name or t3:// path containing the objects to bundle + examples: + - my-bucket + - t3://my-bucket + - name: keys + description: "Comma-separated object keys, or path to a file with one key per line. If a local file matching the value exists, it is read as a keys file. If omitted, reads keys from stdin" + alias: k + - name: output + description: Output file path. Defaults to stdout (for piping) + alias: o + - name: compression + description: Compression algorithm for the archive + options: [none, gzip, zstd] + default: none + - name: on-error + description: How to handle missing objects. 'skip' omits them, 'fail' aborts the request + options: [skip, fail] + default: skip + ######################### # Manage organizations ######################### diff --git a/src/utils/options.ts b/src/utils/options.ts index 79046ed..7e1858a 100644 --- a/src/utils/options.ts +++ b/src/utils/options.ts @@ -73,3 +73,15 @@ export function parseBoolean( if (typeof value === 'boolean') return value; return value === 'true'; } + +/** + * Read all of stdin as a UTF-8 string. + * Use when stdin is piped (i.e. `!process.stdin.isTTY`). + */ +export async function readStdin(): Promise { + const chunks: Buffer[] = []; + for await (const chunk of process.stdin) { + chunks.push(chunk); + } + return Buffer.concat(chunks).toString('utf-8'); +} diff --git a/test/cli.test.ts b/test/cli.test.ts index e0b06cc..864db8b 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -126,6 +126,16 @@ describe('CLI Help Commands', () => { expect(result.stdout).toContain('path'); }); + it('should show bundle help', () => { + const result = runCli('bundle help'); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('bundle'); + expect(result.stdout).toContain('--keys'); + expect(result.stdout).toContain('--output'); + expect(result.stdout).toContain('--compression'); + expect(result.stdout).toContain('--on-error'); + }); + it('should show configure help', () => { const result = runCli('configure help'); expect(result.exitCode).toBe(0); @@ -452,6 +462,129 @@ describe.skipIf(skipTests)('CLI Integration Tests', () => { }); }); + describe('bundle command', () => { + const bundleDir = 'bundle-test'; + const rootTxt = 'bundle-root.txt'; + const rootJson = 'bundle-root.json'; + const nestedTxt = `${bundleDir}/nested.txt`; + const nestedJson = `${bundleDir}/nested.json`; + const tmpDir = join(tmpdir(), `tigris-bundle-test-${Date.now()}`); + + beforeAll(() => { + mkdirSync(tmpDir, { recursive: true }); + + // Create test files at bucket root + const txtFile = join(tmpDir, 'root.txt'); + const jsonFile = join(tmpDir, 'root.json'); + writeFileSync(txtFile, 'hello from txt'); + writeFileSync(jsonFile, JSON.stringify({ hello: 'from json' })); + + runCli(`objects put ${testBucket} ${rootTxt} ${txtFile}`); + runCli(`objects put ${testBucket} ${rootJson} ${jsonFile}`); + + // Create test files in a folder + const nestedTxtFile = join(tmpDir, 'nested.txt'); + const nestedJsonFile = join(tmpDir, 'nested.json'); + writeFileSync(nestedTxtFile, 'nested txt content'); + writeFileSync(nestedJsonFile, JSON.stringify({ nested: true })); + + runCli(`mk ${testBucket}/${bundleDir}/`); + runCli(`objects put ${testBucket} ${nestedTxt} ${nestedTxtFile}`); + runCli(`objects put ${testBucket} ${nestedJson} ${nestedJsonFile}`); + }); + + afterAll(() => { + rmSync(tmpDir, { recursive: true, force: true }); + runCli(`rm ${t3(testBucket)}/${bundleDir} -r -f`); + runCli(`rm ${t3(testBucket)}/${rootTxt} -f`); + runCli(`rm ${t3(testBucket)}/${rootJson} -f`); + }); + + it('should bundle root objects with inline keys', () => { + const output = join(tmpDir, 'root-bundle.tar'); + const result = runCli( + `bundle ${testBucket} --keys ${rootTxt},${rootJson} --output ${output}` + ); + expect(result.exitCode).toBe(0); + expect(existsSync(output)).toBe(true); + + // Verify tar contents + const tarList = execSync(`tar tf ${output}`, { encoding: 'utf-8' }); + expect(tarList).toContain(rootTxt); + expect(tarList).toContain(rootJson); + }); + + it('should bundle with keys from file', () => { + const keysFile = join(tmpDir, 'keys.txt'); + writeFileSync(keysFile, `${rootTxt}\n${rootJson}\n`); + + const output = join(tmpDir, 'from-file.tar'); + const result = runCli( + `bundle ${testBucket} --keys ${keysFile} --output ${output}` + ); + expect(result.exitCode).toBe(0); + + const tarList = execSync(`tar tf ${output}`, { encoding: 'utf-8' }); + expect(tarList).toContain(rootTxt); + expect(tarList).toContain(rootJson); + }); + + it('should bundle nested objects with path prefix', () => { + const output = join(tmpDir, 'nested-bundle.tar'); + const result = runCli( + `bundle ${t3(testBucket)}/${bundleDir} --keys nested.txt,nested.json --output ${output}` + ); + expect(result.exitCode).toBe(0); + + const tarList = execSync(`tar tf ${output}`, { encoding: 'utf-8' }); + expect(tarList).toContain('nested.txt'); + expect(tarList).toContain('nested.json'); + }); + + it('should bundle with gzip compression', () => { + const output = join(tmpDir, 'compressed.tar.gz'); + const result = runCli( + `bundle ${testBucket} --keys ${rootTxt},${rootJson} --output ${output}` + ); + expect(result.exitCode).toBe(0); + + // tar should be able to decompress gzip + const tarList = execSync(`tar tzf ${output}`, { encoding: 'utf-8' }); + expect(tarList).toContain(rootTxt); + expect(tarList).toContain(rootJson); + }); + + it('should bundle with explicit compression flag', () => { + const output = join(tmpDir, 'explicit-gzip.tar'); + const result = runCli( + `bundle ${testBucket} --keys ${rootTxt} --compression gzip --output ${output}` + ); + expect(result.exitCode).toBe(0); + + // Despite .tar extension, content is gzip-compressed + const tarList = execSync(`tar tzf ${output}`, { encoding: 'utf-8' }); + expect(tarList).toContain(rootTxt); + }); + + it('should output JSON with --json flag', () => { + const output = join(tmpDir, 'json-mode.tar'); + const result = runCli( + `bundle ${testBucket} --keys ${rootTxt},${rootJson} --output ${output} --json` + ); + expect(result.exitCode).toBe(0); + + const parsed = JSON.parse(result.stdout.trim()); + expect(parsed.action).toBe('bundled'); + expect(parsed.bucket).toBe(testBucket); + expect(parsed.keys).toBe(2); + }); + + it('should fail with no keys provided', () => { + const result = runCli(`bundle ${testBucket}`); + expect(result.exitCode).not.toBe(0); + }); + }); + describe('folder auto-detection', () => { const autoFolder = 'autodetect'; const copiedFolder = 'copied';