Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/download/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,11 @@ describe('download helpers', () => {
const destPath = path.join(tempDir, 'file.txt');
const result = await httpDownload(`${baseUrl}/loop`, destPath, { maxRedirects: 2 });

expect(result).toEqual({
expect(result).toEqual(expect.objectContaining({
success: false,
size: 0,
error: 'Too many redirects (> 2)',
});
}));
expect(fs.existsSync(destPath)).toBe(false);
});
});
38 changes: 32 additions & 6 deletions src/download/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,15 @@ export interface DownloadOptions {
maxRedirects?: number;
}

export interface HttpDownloadResult {
success: boolean;
size: number;
error?: string;
statusCode?: number;
contentType?: string;
finalUrl?: string;
}

export interface YtdlpOptions {
cookies?: string;
cookiesFile?: string;
Expand Down Expand Up @@ -82,7 +91,7 @@ export async function httpDownload(
destPath: string,
options: DownloadOptions = {},
redirectCount = 0,
): Promise<{ success: boolean; size: number; error?: string }> {
): Promise<HttpDownloadResult> {
const { cookies, headers = {}, timeout = 30000, onProgress, maxRedirects = 10 } = options;

return new Promise((resolve) => {
Expand Down Expand Up @@ -111,7 +120,7 @@ export async function httpDownload(
file.close();
if (fs.existsSync(tempPath)) fs.unlinkSync(tempPath);
if (redirectCount >= maxRedirects) {
resolve({ success: false, size: 0, error: `Too many redirects (> ${maxRedirects})` });
resolve({ success: false, size: 0, error: `Too many redirects (> ${maxRedirects})`, finalUrl: url });
return;
}
httpDownload(
Expand All @@ -126,7 +135,14 @@ export async function httpDownload(
if (response.statusCode !== 200) {
file.close();
if (fs.existsSync(tempPath)) fs.unlinkSync(tempPath);
resolve({ success: false, size: 0, error: `HTTP ${response.statusCode}` });
resolve({
success: false,
size: 0,
error: `HTTP ${response.statusCode}`,
statusCode: response.statusCode,
contentType: normalizeHeaderValue(response.headers['content-type']),
finalUrl: url,
});
return;
}

Expand All @@ -144,21 +160,27 @@ export async function httpDownload(
file.close();
// Rename temp file to final destination
fs.renameSync(tempPath, destPath);
resolve({ success: true, size: received });
resolve({
success: true,
size: received,
statusCode: response.statusCode,
contentType: normalizeHeaderValue(response.headers['content-type']),
finalUrl: url,
});
});
});

request.on('error', (err) => {
file.close();
if (fs.existsSync(tempPath)) fs.unlinkSync(tempPath);
resolve({ success: false, size: 0, error: err.message });
resolve({ success: false, size: 0, error: err.message, finalUrl: url });
});

request.on('timeout', () => {
request.destroy();
file.close();
if (fs.existsSync(tempPath)) fs.unlinkSync(tempPath);
resolve({ success: false, size: 0, error: 'Timeout' });
resolve({ success: false, size: 0, error: 'Timeout', finalUrl: url });
});
});
}
Expand All @@ -167,6 +189,10 @@ export function resolveRedirectUrl(currentUrl: string, location: string): string
return new URL(location, currentUrl).toString();
}

function normalizeHeaderValue(header: string | string[] | undefined): string | undefined {
return Array.isArray(header) ? header[0] : header;
}

/**
* Export cookies to Netscape format for yt-dlp.
*/
Expand Down
26 changes: 26 additions & 0 deletions src/pipeline/executor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import { describe, it, expect, vi } from 'vitest';
import { executePipeline } from './index.js';
import { getStep, registerStep } from './registry.js';
import { ConfigError } from '../errors.js';
import type { IPage } from '../types.js';

Expand Down Expand Up @@ -189,4 +190,29 @@ describe('executePipeline', () => {
expect(result).toEqual([{ a: 1 }]);
expect(page.goto).toHaveBeenCalledWith('https://example.com');
});

it.each(['intercept', 'tap'])('retries transient browser errors for %s step', async (stepName) => {
const original = getStep(stepName);
expect(original).toBeDefined();

let attempts = 0;
registerStep(stepName, async () => {
attempts += 1;
if (attempts === 1) {
throw new Error('Extension disconnected');
}
return 'ok';
});

try {
const result = await executePipeline(null, [
{ [stepName]: {} },
]);

expect(result).toBe('ok');
expect(attempts).toBe(2);
} finally {
registerStep(stepName, original!);
}
});
});
2 changes: 1 addition & 1 deletion src/pipeline/executor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ export interface PipelineContext {
}

/** Steps that interact with the browser and may fail transiently */
const BROWSER_STEPS = new Set(['navigate', 'evaluate', 'click', 'type', 'press', 'wait', 'snapshot', 'scroll']);
const BROWSER_STEPS = new Set(['navigate', 'evaluate', 'click', 'type', 'press', 'wait', 'snapshot', 'scroll', 'intercept', 'tap', 'download']);

export async function executePipeline(
page: IPage | null,
Expand Down
Loading