Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Highlights

- Added a hardened outbound profile for cluster and data-plane deployments
- Ambient proxy environment variables are now ignored by default
- Added hostname, port, and redirect restrictions for tighter egress policy

### What's Changed

- `fix(security): harden outbound fetch policy and add deployment guidance`

## [0.1.3] - 2026-03-12

### Highlights
Expand Down
20 changes: 19 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ fetchkit fetch https://example.com -o json
# Custom user agent
fetchkit fetch https://example.com --user-agent "MyBot/1.0"

# Hardened outbound policy for cluster/data-plane use
fetchkit fetch https://example.com --hardened

# Show full documentation
fetchkit --llmtxt
```
Expand Down Expand Up @@ -85,6 +88,9 @@ Run as a Model Context Protocol server:

```bash
fetchkit mcp

# Hardened profile for cluster/data-plane use
fetchkit mcp --hardened
```

Exposes `fetchkit` tool over JSON-RPC 2.0 stdio transport. Returns markdown with frontmatter (same format as CLI). Compatible with Claude Desktop and other MCP clients.
Expand Down Expand Up @@ -129,6 +135,17 @@ let request = FetchRequest::new("https://example.com");
let response = tool.execute(request).await.unwrap();
```

### Hardened Tool Profile

```rust
use fetchkit::Tool;

let tool = Tool::builder()
.hardened()
.allow_prefix("https://docs.example.com")
.build();
```

## Python Bindings

```bash
Expand Down Expand Up @@ -197,9 +214,10 @@ let tool = Tool::builder()

DNS pinning prevents DNS rebinding attacks. IPv6-mapped IPv4 addresses are canonicalized before validation.
Redirects are followed manually in the default fetcher so each hop is revalidated against scheme and DNS policy. Allow/block prefixes are matched against parsed URLs rather than raw strings, which prevents lookalike host overmatches such as `allowed.example.com.evil.test`.
Proxy environment variables are ignored by default; opt in with `ToolBuilder::respect_proxy_env(true)` only when you intentionally want `HTTP_PROXY`/`HTTPS_PROXY` routing.
Proxy environment variables are ignored by default. Use the hardened profile for cluster-facing deployments and opt in with `ToolBuilder::respect_proxy_env(true)` only when it is part of an intentional egress design.

See [`specs/threat-model.md`](specs/threat-model.md) for the full threat model.
See [`docs/hardening.md`](docs/hardening.md) for deployment guidance.

## Configuration

Expand Down
57 changes: 47 additions & 10 deletions crates/fetchkit-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,15 @@ struct Cli {
#[derive(Subcommand, Debug)]
enum Commands {
/// Run as MCP (Model Context Protocol) server over stdio
Mcp,
Mcp {
/// Apply the hardened outbound policy profile
#[arg(long)]
hardened: bool,

/// Allow HTTP_PROXY/HTTPS_PROXY/NO_PROXY from the environment
#[arg(long)]
allow_env_proxy: bool,
},
/// Fetch URL and output as markdown with metadata frontmatter
Fetch {
/// URL to fetch
Expand All @@ -56,6 +64,14 @@ enum Commands {
/// Custom User-Agent
#[arg(long)]
user_agent: Option<String>,

/// Apply the hardened outbound policy profile
#[arg(long)]
hardened: bool,

/// Allow HTTP_PROXY/HTTPS_PROXY/NO_PROXY from the environment
#[arg(long)]
allow_env_proxy: bool,
},
}

Expand All @@ -70,15 +86,20 @@ async fn main() {
}

match cli.command {
Some(Commands::Mcp) => {
mcp::run_server().await;
Some(Commands::Mcp {
hardened,
allow_env_proxy,
}) => {
mcp::run_server(build_tool(None, hardened, allow_env_proxy)).await;
}
Some(Commands::Fetch {
url,
output,
user_agent,
hardened,
allow_env_proxy,
}) => {
run_fetch(&url, output, user_agent).await;
run_fetch(&url, output, user_agent, hardened, allow_env_proxy).await;
}
None => {
eprintln!("Usage: fetchkit fetch <URL>");
Expand All @@ -89,18 +110,34 @@ async fn main() {
}
}

async fn run_fetch(url: &str, output: OutputFormat, user_agent: Option<String>) {
// Build request with markdown conversion
let request = FetchRequest::new(url).as_markdown();

// Build tool
fn build_tool(user_agent: Option<String>, hardened: bool, allow_env_proxy: bool) -> Tool {
let mut builder = Tool::builder().enable_markdown(true);

if hardened {
builder = builder.hardened();
}

if allow_env_proxy {
builder = builder.use_env_proxy(true);
}

if let Some(ua) = user_agent {
builder = builder.user_agent(ua);
}

let tool = builder.build();
builder.build()
}

async fn run_fetch(
url: &str,
output: OutputFormat,
user_agent: Option<String>,
hardened: bool,
allow_env_proxy: bool,
) {
// Build request with markdown conversion
let request = FetchRequest::new(url).as_markdown();
let tool = build_tool(user_agent, hardened, allow_env_proxy);

// Execute request
match tool.execute(request).await {
Expand Down
10 changes: 4 additions & 6 deletions crates/fetchkit-cli/src/mcp.rs
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,8 @@ struct McpServer {
}

impl McpServer {
fn new() -> Self {
Self {
tool: Tool::default(),
}
fn new(tool: Tool) -> Self {
Self { tool }
}

async fn handle_request(&self, request: JsonRpcRequest) -> JsonRpcResponse {
Expand Down Expand Up @@ -222,8 +220,8 @@ fn format_md_with_frontmatter(response: &fetchkit::FetchResponse) -> String {
}

/// Run the MCP server over stdio
pub async fn run_server() {
let server = McpServer::new();
pub async fn run_server(tool: Tool) {
let server = McpServer::new(tool);
let stdin = io::stdin();
let mut stdout = io::stdout();

Expand Down
26 changes: 26 additions & 0 deletions crates/fetchkit-cli/tests/cli_integration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,32 @@ fn test_help_flag() {
assert!(stdout.contains("fetch") || stdout.contains("mcp"));
}

#[test]
fn test_fetch_help_lists_hardening_flags() {
let output = Command::new(fetchkit_bin())
.args(["fetch", "--help"])
.output()
.expect("failed to run fetchkit");

let stdout = String::from_utf8_lossy(&output.stdout);
assert!(output.status.success());
assert!(stdout.contains("--hardened"));
assert!(stdout.contains("--allow-env-proxy"));
}

#[test]
fn test_mcp_help_lists_hardening_flags() {
let output = Command::new(fetchkit_bin())
.args(["mcp", "--help"])
.output()
.expect("failed to run fetchkit");

let stdout = String::from_utf8_lossy(&output.stdout);
assert!(output.status.success());
assert!(stdout.contains("--hardened"));
assert!(stdout.contains("--allow-env-proxy"));
}

// ============================================================================
// --version flag
// ============================================================================
Expand Down
49 changes: 47 additions & 2 deletions crates/fetchkit-python/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -177,21 +177,48 @@ pub struct PyFetchKitTool {
impl PyFetchKitTool {
/// Create a new tool with default options
#[new]
#[pyo3(signature = (enable_markdown=true, enable_text=true, user_agent=None, allow_prefixes=None, block_prefixes=None, max_body_size=None, respect_proxy_env=false))]
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (
enable_markdown=true,
enable_text=true,
user_agent=None,
allow_prefixes=None,
block_prefixes=None,
max_body_size=None,
block_private_ips=true,
respect_proxy_env=false,
allowed_ports=None,
blocked_hosts=None,
same_host_redirects_only=false,
hardened=false
))]
fn new(
enable_markdown: bool,
enable_text: bool,
user_agent: Option<String>,
allow_prefixes: Option<Vec<String>>,
block_prefixes: Option<Vec<String>>,
max_body_size: Option<usize>,
block_private_ips: bool,
respect_proxy_env: bool,
allowed_ports: Option<Vec<u16>>,
blocked_hosts: Option<Vec<String>>,
same_host_redirects_only: bool,
hardened: bool,
) -> PyResult<Self> {
let mut builder = ToolBuilder::new()
.enable_markdown(enable_markdown)
.enable_text(enable_text)
.respect_proxy_env(respect_proxy_env);

if hardened {
builder = builder.hardened();
}

builder = builder
.block_private_ips(block_private_ips)
.same_host_redirects_only(same_host_redirects_only);

if let Some(ua) = user_agent {
builder = builder.user_agent(ua);
}
Expand All @@ -212,6 +239,22 @@ impl PyFetchKitTool {
builder = builder.max_body_size(max_bytes);
}

if let Some(ports) = allowed_ports {
for port in ports {
builder = builder.allow_port(port);
}
}

if let Some(hosts) = blocked_hosts {
for host in hosts {
builder = if host.starts_with('.') {
builder.block_host_suffix(host)
} else {
builder.block_host(host)
};
}
}

let runtime = tokio::runtime::Runtime::new()
.map_err(|e| PyValueError::new_err(format!("Failed to create runtime: {}", e)))?;

Expand Down Expand Up @@ -280,7 +323,9 @@ fn fetch(
as_markdown: Option<bool>,
as_text: Option<bool>,
) -> PyResult<PyFetchResponse> {
let tool = PyFetchKitTool::new(true, true, None, None, None, None, false)?;
let tool = PyFetchKitTool::new(
true, true, None, None, None, None, true, false, None, None, false, false,
)?;
tool.fetch(url, method, as_markdown, as_text)
}

Expand Down
Loading
Loading