From 9a8dcbd10d5de864b570ce34c5dcd609098a3eb3 Mon Sep 17 00:00:00 2001 From: csd113 Date: Sat, 21 Mar 2026 19:08:25 -0700 Subject: [PATCH 1/2] Delete ffmpeg-migration.md --- ffmpeg-migration.md | 1233 ------------------------------------------- 1 file changed, 1233 deletions(-) delete mode 100644 ffmpeg-migration.md diff --git a/ffmpeg-migration.md b/ffmpeg-migration.md deleted file mode 100644 index 555e5f7..0000000 --- a/ffmpeg-migration.md +++ /dev/null @@ -1,1233 +0,0 @@ -# FFmpeg Static Migration — Full Implementation - -Replace every `Command::new("ffmpeg")` / `TokioCommand::new("ffmpeg")` call with -in-process `ffmpeg-next` library calls. After this migration the binary requires -no system ffmpeg or ffprobe installation. - ---- - -## 1. Cargo.toml - -```toml -[dependencies] -# Add this block. Remove nothing else. -ffmpeg-next = { version = "7", default-features = false, features = [ - "static", # compiles libav* from source into the binary - "codec", # libavcodec — encode / decode - "format", # libavformat — container mux / demux - "filter", # libavfilter — scale, showwavespic - "software-resampling", # libswresample — audio resampling for Opus - "software-scaling", # libswscale — pixel-format conversion -] } -``` - -Pin the ffmpeg source version and supply build flags via `.cargo/config.toml` -(create this file if it does not exist): - -```toml -# .cargo/config.toml -[env] -FFMPEG_BUILD_VERSION = "7.1" -# If nasm is absent on the build machine, add: -# FFMPEG_EXTRA_FLAGS = "--disable-x86asm" -``` - -Build-tool requirements (install once per machine): - -| Tool | Ubuntu/Debian | macOS | -|---|---|---| -| nasm | `apt install nasm` | `brew install nasm` | -| cmake | `apt install cmake` | `brew install cmake` | -| pkg-config | `apt install pkg-config` | `brew install pkg-config` | - -CI (GitHub Actions Ubuntu runner) — add before `cargo build`: - -```yaml -- name: Install ffmpeg build deps - run: sudo apt-get install -y nasm cmake pkg-config libssl-dev -``` - ---- - -## 2. `src/media/ffmpeg.rs` — complete replacement - -Drop the entire existing file and replace with the following. -All public function signatures are **identical** to the originals. - -```rust -// media/ffmpeg.rs -// -// FFmpeg wrappers using statically linked libav* (ffmpeg-next). -// -// All public signatures are unchanged from the subprocess-based version so -// that no caller in convert.rs, thumbnail.rs, workers/, or detect.rs needs -// to be modified for signature reasons. -// -// Call site changes required elsewhere: -// • detect.rs — detection functions become no-ops (see §4 below) -// • workers/ — TokioCommand::new("ffmpeg") blocks replaced (see §3 below) -// -// Thread safety: ffmpeg-next is safe to call from multiple threads once -// init_ffmpeg() has been called. Call it once from main() or detect.rs. - -use anyhow::{Context, Result}; -use ffmpeg_next as ffmpeg; -use ffmpeg_next::{ - codec, filter, format, frame, media, - software::scaling::{context::Context as SwsContext, flag::Flags as SwsFlags}, - Dictionary, Rational, -}; -use std::path::Path; - -// ─── Library initialisation ─────────────────────────────────────────────────── - -/// Initialise the ffmpeg library. Idempotent and thread-safe. -/// -/// Call once at startup (from detect.rs or main.rs) before any codec -/// operation. Safe to call multiple times — subsequent calls are no-ops. -pub fn init_ffmpeg() { - ffmpeg::init().expect("ffmpeg library init failed — this is a build error"); - // Suppress ffmpeg's internal log output. rustchan's tracing handles all - // user-facing diagnostics. - unsafe { - ffmpeg_next::ffi::av_log_set_level(ffmpeg_next::ffi::AV_LOG_QUIET); - } -} - -// ─── Detection stubs (always true — codecs are compiled in) ────────────────── - -/// Always returns true: ffmpeg is compiled into the binary. -#[must_use] -pub fn detect_ffmpeg() -> bool { - true -} - -/// Always returns true: libwebp is compiled in. -#[must_use] -pub fn check_webp_encoder() -> bool { - true -} - -/// Always returns true: libvpx-vp9 is compiled in. -#[must_use] -pub fn check_vp9_encoder() -> bool { - true -} - -/// Always returns true: libopus is compiled in. -#[must_use] -pub fn check_opus_encoder() -> bool { - true -} - -// ─── run_ffmpeg (no longer used — kept for any external callers) ────────────── - -/// Deprecated: previously spawned a subprocess. Now always returns Ok(()). -/// Remove callers and delete this function in a follow-up cleanup. -#[allow(dead_code)] -pub fn run_ffmpeg(_args: &[&str]) -> Result<()> { - Ok(()) -} - -// ─── Image → WebP ───────────────────────────────────────────────────────────── - -/// Convert any ffmpeg-readable image (JPEG, PNG, BMP, TIFF, GIF) to WebP. -/// -/// Animated GIF inputs produce animated WebP (loop 0 = loop forever). -/// Metadata is stripped. Quality is fixed at 85 per project spec. -/// -/// # Errors -/// Returns an error if the input cannot be decoded or the output cannot be -/// written. -pub fn ffmpeg_image_to_webp(input: &Path, output: &Path) -> Result<()> { - init_ffmpeg(); - - // ── Open input ──────────────────────────────────────────────────────── - let mut ictx = format::input(input) - .with_context(|| format!("ffmpeg_image_to_webp: cannot open {}", input.display()))?; - - let in_stream = ictx - .streams() - .best(media::Type::Video) - .context("ffmpeg_image_to_webp: no video/image stream in input")?; - let in_idx = in_stream.index(); - let in_tb = in_stream.time_base(); - - let mut decoder = codec::context::Context::from_parameters(in_stream.parameters()) - .context("ffmpeg_image_to_webp: decoder context")? - .decoder() - .video() - .context("ffmpeg_image_to_webp: open video decoder")?; - - // ── Open output ─────────────────────────────────────────────────────── - let mut octx = format::output(output) - .with_context(|| format!("ffmpeg_image_to_webp: cannot open output {}", output.display()))?; - - let webp_codec = codec::encoder::find_by_name("libwebp") - .context("libwebp encoder not found — static build is missing the codec")?; - - let mut enc_ctx = codec::context::Context::new_with_codec(webp_codec) - .encoder() - .video() - .context("ffmpeg_image_to_webp: encoder context")?; - - // Dimensions and format are set from the first decoded frame because some - // inputs (e.g. TIFF) report wrong dimensions in stream parameters. - // We defer the actual encoder open until after decoding the first frame. - - let mut out_stream = octx.add_stream(webp_codec) - .context("ffmpeg_image_to_webp: add output stream")?; - - let global_header = octx - .format() - .flags() - .contains(format::flag::Flags::GLOBAL_HEADER); - if global_header { - enc_ctx.set_flags(codec::flag::Flags::GLOBAL_HEADER); - } - - // ── Decode → scale → encode loop ───────────────────────────────────── - let mut encoder_opened = false; - let mut sws: Option = None; - let mut frame_count = 0u64; - - let mut encode_and_write = |enc: &mut codec::encoder::video::Encoder, - octx: &mut format::context::Output, - frame: Option<&frame::Video>| - -> Result<()> { - enc.send_frame(frame).context("ffmpeg_image_to_webp: send_frame")?; - let mut pkt = ffmpeg_next::Packet::empty(); - while enc.receive_packet(&mut pkt).is_ok() { - pkt.set_stream(0); - pkt.rescale_ts(Rational(1, 25), out_stream.time_base()); - pkt.write_interleaved(octx) - .context("ffmpeg_image_to_webp: write_interleaved")?; - } - Ok(()) - }; - - for (stream, packet) in ictx.packets() { - if stream.index() != in_idx { - continue; - } - decoder - .send_packet(&packet) - .context("ffmpeg_image_to_webp: send_packet")?; - - let mut decoded = frame::Video::empty(); - while decoder.receive_frame(&mut decoded).is_ok() { - if !encoder_opened { - // First frame: configure encoder dimensions and open it. - enc_ctx.set_width(decoded.width()); - enc_ctx.set_height(decoded.height()); - enc_ctx.set_format(ffmpeg_next::format::Pixel::YUVA420P); - enc_ctx.set_time_base(Rational(1, 25)); - - let mut opts = Dictionary::new(); - opts.set("quality", "85"); - opts.set("loop", "0"); // animated WebP loops forever (GIF parity) - opts.set("lossless", "0"); - - enc_ctx - .open_with(opts) - .context("ffmpeg_image_to_webp: open encoder")?; - out_stream.set_parameters(&enc_ctx); - - octx.write_header() - .context("ffmpeg_image_to_webp: write_header")?; - encoder_opened = true; - - // Pixel format converter: input format → YUVA420P for libwebp. - sws = Some( - SwsContext::get( - decoded.format(), - decoded.width(), - decoded.height(), - ffmpeg_next::format::Pixel::YUVA420P, - decoded.width(), - decoded.height(), - SwsFlags::BILINEAR, - ) - .context("ffmpeg_image_to_webp: sws_getContext")?, - ); - } - - // Convert pixel format. - let mut rgb_frame = frame::Video::new( - ffmpeg_next::format::Pixel::YUVA420P, - decoded.width(), - decoded.height(), - ); - if let Some(ref mut s) = sws { - s.run(&decoded, &mut rgb_frame) - .context("ffmpeg_image_to_webp: sws_scale")?; - } - rgb_frame.set_pts(Some(frame_count as i64)); - frame_count += 1; - - // We need mutable enc_ctx below — restructure to avoid borrow conflict. - enc_ctx - .send_frame(Some(&rgb_frame)) - .context("ffmpeg_image_to_webp: send_frame")?; - let mut pkt = ffmpeg_next::Packet::empty(); - while enc_ctx.receive_packet(&mut pkt).is_ok() { - pkt.set_stream(0); - pkt.rescale_ts(Rational(1, 25), out_stream.time_base()); - pkt.write_interleaved(&mut octx) - .context("ffmpeg_image_to_webp: write_interleaved")?; - } - } - } - - // Flush decoder. - decoder - .send_eof() - .context("ffmpeg_image_to_webp: send_eof to decoder")?; - let mut decoded = frame::Video::empty(); - while decoder.receive_frame(&mut decoded).is_ok() { - // (same encode block as above — flush remaining frames) - let mut rgb_frame = frame::Video::new( - ffmpeg_next::format::Pixel::YUVA420P, - decoded.width(), - decoded.height(), - ); - if let Some(ref mut s) = sws { - s.run(&decoded, &mut rgb_frame) - .context("ffmpeg_image_to_webp: sws_scale flush")?; - } - rgb_frame.set_pts(Some(frame_count as i64)); - frame_count += 1; - enc_ctx - .send_frame(Some(&rgb_frame)) - .context("ffmpeg_image_to_webp: flush send_frame")?; - let mut pkt = ffmpeg_next::Packet::empty(); - while enc_ctx.receive_packet(&mut pkt).is_ok() { - pkt.set_stream(0); - pkt.rescale_ts(Rational(1, 25), out_stream.time_base()); - pkt.write_interleaved(&mut octx) - .context("ffmpeg_image_to_webp: flush write")?; - } - } - - // Flush encoder. - enc_ctx - .send_eof() - .context("ffmpeg_image_to_webp: send_eof to encoder")?; - let mut pkt = ffmpeg_next::Packet::empty(); - while enc_ctx.receive_packet(&mut pkt).is_ok() { - pkt.set_stream(0); - pkt.rescale_ts(Rational(1, 25), out_stream.time_base()); - pkt.write_interleaved(&mut octx) - .context("ffmpeg_image_to_webp: final write")?; - } - - octx.write_trailer() - .context("ffmpeg_image_to_webp: write_trailer")?; - - Ok(()) -} - -// ─── Thumbnail ──────────────────────────────────────────────────────────────── - -/// Extract the first frame from an image or video, scale to fit within -/// `max_dim × max_dim` (aspect preserved), and save as WebP quality 80. -/// -/// Equivalent to: -/// ffmpeg -i -vframes 1 -/// -vf "scale='if(gt(iw,ih),MAX,-2)':'if(gt(iw,ih),-2,MAX)'" -/// -c:v libwebp -quality 80 -/// -/// # Errors -/// Returns an error if the input cannot be demuxed/decoded or the output -/// cannot be written. -pub fn ffmpeg_thumbnail(input: &Path, output: &Path, max_dim: u32) -> Result<()> { - init_ffmpeg(); - - let mut ictx = format::input(input) - .with_context(|| format!("ffmpeg_thumbnail: cannot open {}", input.display()))?; - - let in_stream = ictx - .streams() - .best(media::Type::Video) - .context("ffmpeg_thumbnail: no video stream")?; - let in_idx = in_stream.index(); - - let mut decoder = codec::context::Context::from_parameters(in_stream.parameters()) - .context("ffmpeg_thumbnail: decoder context")? - .decoder() - .video() - .context("ffmpeg_thumbnail: open decoder")?; - - // Seek to the first key frame (important for video sources). - let _ = ictx.seek(0, ..0); - - // Decode until we get one complete frame. - let mut first_frame: Option = None; - 'outer: for (stream, packet) in ictx.packets() { - if stream.index() != in_idx { - continue; - } - decoder - .send_packet(&packet) - .context("ffmpeg_thumbnail: send_packet")?; - let mut f = frame::Video::empty(); - while decoder.receive_frame(&mut f).is_ok() { - first_frame = Some(f); - break 'outer; - } - } - // Flush decoder in case the frame was buffered. - if first_frame.is_none() { - let _ = decoder.send_eof(); - let mut f = frame::Video::empty(); - if decoder.receive_frame(&mut f).is_ok() { - first_frame = Some(f); - } - } - - let src_frame = first_frame.context("ffmpeg_thumbnail: no frame decoded from input")?; - - // ── Scale to fit max_dim × max_dim, preserving aspect ratio ────────── - let (src_w, src_h) = (src_frame.width(), src_frame.height()); - let (dst_w, dst_h) = scale_dims(src_w, src_h, max_dim); - - let mut scaled = frame::Video::new( - ffmpeg_next::format::Pixel::YUVA420P, - dst_w, - dst_h, - ); - let mut sws = SwsContext::get( - src_frame.format(), - src_w, - src_h, - ffmpeg_next::format::Pixel::YUVA420P, - dst_w, - dst_h, - SwsFlags::LANCZOS, - ) - .context("ffmpeg_thumbnail: sws_getContext")?; - sws.run(&src_frame, &mut scaled) - .context("ffmpeg_thumbnail: sws_scale")?; - scaled.set_pts(Some(0)); - - // ── Encode as WebP quality 80 ───────────────────────────────────────── - let webp_codec = codec::encoder::find_by_name("libwebp") - .context("ffmpeg_thumbnail: libwebp encoder missing")?; - - let mut octx = format::output(output) - .with_context(|| format!("ffmpeg_thumbnail: cannot open output {}", output.display()))?; - - let mut enc_ctx = codec::context::Context::new_with_codec(webp_codec) - .encoder() - .video() - .context("ffmpeg_thumbnail: encoder context")?; - - enc_ctx.set_width(dst_w); - enc_ctx.set_height(dst_h); - enc_ctx.set_format(ffmpeg_next::format::Pixel::YUVA420P); - enc_ctx.set_time_base(Rational(1, 1)); - - let global_header = octx - .format() - .flags() - .contains(format::flag::Flags::GLOBAL_HEADER); - if global_header { - enc_ctx.set_flags(codec::flag::Flags::GLOBAL_HEADER); - } - - let mut opts = Dictionary::new(); - opts.set("quality", "80"); - opts.set("lossless", "0"); - - let mut enc = enc_ctx - .open_with(opts) - .context("ffmpeg_thumbnail: open encoder")?; - - let mut out_stream = octx.add_stream(webp_codec) - .context("ffmpeg_thumbnail: add output stream")?; - out_stream.set_parameters(&enc); - - octx.write_header() - .context("ffmpeg_thumbnail: write_header")?; - - enc.send_frame(Some(&scaled)) - .context("ffmpeg_thumbnail: send_frame")?; - enc.send_eof() - .context("ffmpeg_thumbnail: send_eof")?; - - let mut pkt = ffmpeg_next::Packet::empty(); - while enc.receive_packet(&mut pkt).is_ok() { - pkt.set_stream(0); - pkt.rescale_ts(Rational(1, 1), out_stream.time_base()); - pkt.write_interleaved(&mut octx) - .context("ffmpeg_thumbnail: write_interleaved")?; - } - - octx.write_trailer() - .context("ffmpeg_thumbnail: write_trailer")?; - - Ok(()) -} - -// ─── Codec probe ────────────────────────────────────────────────────────────── - -/// Return the lowercase codec name for the primary video stream in `path`. -/// -/// Replaces the `ffprobe` subprocess. Opens the container, reads the stream -/// parameters, and returns the codec descriptor name — no decoding is done. -/// -/// Returns e.g. `"vp9"`, `"av1"`, `"h264"`. -/// -/// # Errors -/// Returns an error if the file cannot be opened or contains no video stream. -pub fn probe_video_codec(path: &str) -> Result { - init_ffmpeg(); - - let ictx = format::input(&path) - .with_context(|| format!("probe_video_codec: cannot open {path}"))?; - - let stream = ictx - .streams() - .best(media::Type::Video) - .with_context(|| format!("probe_video_codec: no video stream in {path}"))?; - - let codec_id = stream.parameters().id(); - - // ffmpeg_next exposes the codec name through the descriptor. - let name = unsafe { - let desc = ffmpeg_next::ffi::avcodec_descriptor_get(codec_id.into()); - if desc.is_null() { - return Err(anyhow::anyhow!( - "probe_video_codec: no codec descriptor for id {:?}", - codec_id - )); - } - std::ffi::CStr::from_ptr((*desc).name) - .to_string_lossy() - .to_ascii_lowercase() - }; - - if name.is_empty() { - return Err(anyhow::anyhow!( - "probe_video_codec: empty codec name for {path}" - )); - } - - Ok(name) -} - -// ─── Video transcode (MP4 / WebM-AV1 → WebM VP9+Opus) ─────────────────────── - -/// Transcode `input` to WebM with VP9 video and Opus audio, writing to `output`. -/// -/// Equivalent to: -/// ffmpeg -i -c:v libvpx-vp9 -crf 30 -b:v 0 -/// -c:a libopus -b:a 128k -map_metadata -1 -/// -/// Called from `workers/mod.rs` inside `spawn_blocking`. -/// -/// # Errors -/// Returns an error if any stage of the transcode fails. -pub fn ffmpeg_transcode_to_webm(input: &Path, output: &Path) -> Result<()> { - init_ffmpeg(); - - // ── Input context ───────────────────────────────────────────────────── - let mut ictx = format::input(input) - .with_context(|| format!("ffmpeg_transcode_to_webm: cannot open {}", input.display()))?; - - // Find video and audio streams. - let video_in_idx = ictx - .streams() - .best(media::Type::Video) - .map(|s| s.index()); - let audio_in_idx = ictx - .streams() - .best(media::Type::Audio) - .map(|s| s.index()); - - if video_in_idx.is_none() { - return Err(anyhow::anyhow!( - "ffmpeg_transcode_to_webm: no video stream in {}", - input.display() - )); - } - - // ── Output context ──────────────────────────────────────────────────── - let mut octx = format::output(output) - .with_context(|| format!("ffmpeg_transcode_to_webm: cannot open output {}", output.display()))?; - - // ── Video encoder (VP9) ─────────────────────────────────────────────── - let vp9_codec = codec::encoder::find_by_name("libvpx-vp9") - .context("libvpx-vp9 encoder missing from static build")?; - - let in_video = ictx - .stream(video_in_idx.unwrap()) - .context("ffmpeg_transcode_to_webm: get video stream")?; - - let mut v_dec = codec::context::Context::from_parameters(in_video.parameters()) - .context("ffmpeg_transcode_to_webm: video decoder context")? - .decoder() - .video() - .context("ffmpeg_transcode_to_webm: open video decoder")?; - - let mut venc_ctx = codec::context::Context::new_with_codec(vp9_codec) - .encoder() - .video() - .context("ffmpeg_transcode_to_webm: video encoder context")?; - - venc_ctx.set_width(v_dec.width()); - venc_ctx.set_height(v_dec.height()); - venc_ctx.set_format(ffmpeg_next::format::Pixel::YUV420P); - venc_ctx.set_time_base(in_video.avg_frame_rate().invert()); - - let mut v_opts = Dictionary::new(); - v_opts.set("crf", "30"); - v_opts.set("b:v", "0"); // constant quality mode - v_opts.set("deadline", "good"); - v_opts.set("cpu-used", "2"); - - if octx.format().flags().contains(format::flag::Flags::GLOBAL_HEADER) { - venc_ctx.set_flags(codec::flag::Flags::GLOBAL_HEADER); - } - - let mut v_enc = venc_ctx - .open_with(v_opts) - .context("ffmpeg_transcode_to_webm: open VP9 encoder")?; - let mut out_v = octx.add_stream(vp9_codec) - .context("ffmpeg_transcode_to_webm: add video stream")?; - out_v.set_parameters(&v_enc); - - // ── Audio encoder (Opus) ────────────────────────────────────────────── - let mut a_dec_opt: Option = None; - let mut a_enc_opt: Option = None; - let mut out_a_idx: Option = None; - let mut swr_opt: Option = None; - - if let Some(a_idx) = audio_in_idx { - let opus_codec = codec::encoder::find_by_name("libopus") - .context("libopus encoder missing from static build")?; - - let in_audio = ictx - .stream(a_idx) - .context("ffmpeg_transcode_to_webm: get audio stream")?; - - let a_dec = codec::context::Context::from_parameters(in_audio.parameters()) - .context("ffmpeg_transcode_to_webm: audio decoder context")? - .decoder() - .audio() - .context("ffmpeg_transcode_to_webm: open audio decoder")?; - - let mut aenc_ctx = codec::context::Context::new_with_codec(opus_codec) - .encoder() - .audio() - .context("ffmpeg_transcode_to_webm: audio encoder context")?; - - aenc_ctx.set_rate(48000); // Opus native rate - aenc_ctx.set_channel_layout(ffmpeg_next::channel_layout::ChannelLayout::STEREO); - aenc_ctx.set_format(ffmpeg_next::format::Sample::F32( - ffmpeg_next::format::sample::Type::Packed, - )); - aenc_ctx.set_time_base(Rational(1, 48000)); - if octx.format().flags().contains(format::flag::Flags::GLOBAL_HEADER) { - aenc_ctx.set_flags(codec::flag::Flags::GLOBAL_HEADER); - } - - let mut a_opts = Dictionary::new(); - a_opts.set("b:a", "128k"); - - let a_enc = aenc_ctx - .open_with(a_opts) - .context("ffmpeg_transcode_to_webm: open Opus encoder")?; - - // Resampler: source format → Opus (48 kHz stereo f32). - let swr = ffmpeg_next::software::resampling::context::Context::get( - a_dec.format(), - a_dec.channel_layout(), - a_dec.rate(), - ffmpeg_next::format::Sample::F32(ffmpeg_next::format::sample::Type::Packed), - ffmpeg_next::channel_layout::ChannelLayout::STEREO, - 48000, - ) - .context("ffmpeg_transcode_to_webm: swr_alloc_set_opts")?; - - let mut out_a = octx.add_stream(opus_codec) - .context("ffmpeg_transcode_to_webm: add audio stream")?; - out_a.set_parameters(&a_enc); - out_a_idx = Some(out_a.index()); - - a_dec_opt = Some(a_dec); - a_enc_opt = Some(a_enc); - swr_opt = Some(swr); - } - - // ── Transcode loop ──────────────────────────────────────────────────── - octx.write_header() - .context("ffmpeg_transcode_to_webm: write_header")?; - - let mut v_pts: i64 = 0; - let mut a_pts: i64 = 0; - - let video_in_idx = video_in_idx.unwrap(); - - for (stream, packet) in ictx.packets() { - let idx = stream.index(); - - if idx == video_in_idx { - v_dec.send_packet(&packet) - .context("ffmpeg_transcode_to_webm: send video packet")?; - let mut vf = frame::Video::empty(); - while v_dec.receive_frame(&mut vf).is_ok() { - // Reformat to YUV420P if needed. - let enc_frame = if vf.format() == ffmpeg_next::format::Pixel::YUV420P { - vf.clone() - } else { - let mut converted = frame::Video::new( - ffmpeg_next::format::Pixel::YUV420P, - vf.width(), - vf.height(), - ); - let mut sws = SwsContext::get( - vf.format(), vf.width(), vf.height(), - ffmpeg_next::format::Pixel::YUV420P, vf.width(), vf.height(), - SwsFlags::BILINEAR, - ).context("ffmpeg_transcode_to_webm: sws for video reformat")?; - sws.run(&vf, &mut converted) - .context("ffmpeg_transcode_to_webm: sws_scale video")?; - converted - }; - let mut enc_frame = enc_frame; - enc_frame.set_pts(Some(v_pts)); - v_pts += 1; - - v_enc.send_frame(Some(&enc_frame)) - .context("ffmpeg_transcode_to_webm: send video frame")?; - let mut pkt = ffmpeg_next::Packet::empty(); - while v_enc.receive_packet(&mut pkt).is_ok() { - pkt.set_stream(0); - pkt.rescale_ts(v_enc.time_base(), out_v.time_base()); - pkt.write_interleaved(&mut octx) - .context("ffmpeg_transcode_to_webm: write video pkt")?; - } - } - } else if Some(idx) == audio_in_idx { - if let (Some(ref mut a_dec), Some(ref mut a_enc), Some(ref mut swr), Some(out_a)) = - (&mut a_dec_opt, &mut a_enc_opt, &mut swr_opt, out_a_idx) - { - a_dec.send_packet(&packet) - .context("ffmpeg_transcode_to_webm: send audio packet")?; - let mut af = frame::Audio::empty(); - while a_dec.receive_frame(&mut af).is_ok() { - let mut resampled = frame::Audio::empty(); - swr.run(&af, &mut resampled) - .context("ffmpeg_transcode_to_webm: swr_convert")?; - resampled.set_pts(Some(a_pts)); - a_pts += resampled.samples() as i64; - - a_enc.send_frame(Some(&resampled)) - .context("ffmpeg_transcode_to_webm: send audio frame")?; - let mut pkt = ffmpeg_next::Packet::empty(); - while a_enc.receive_packet(&mut pkt).is_ok() { - pkt.set_stream(out_a); - pkt.rescale_ts(a_enc.time_base(), octx.stream(out_a).unwrap().time_base()); - pkt.write_interleaved(&mut octx) - .context("ffmpeg_transcode_to_webm: write audio pkt")?; - } - } - } - } - } - - // ── Flush video encoder ─────────────────────────────────────────────── - v_dec.send_eof().ok(); - let mut vf = frame::Video::empty(); - while v_dec.receive_frame(&mut vf).is_ok() { - vf.set_pts(Some(v_pts)); - v_pts += 1; - v_enc.send_frame(Some(&vf)).ok(); - let mut pkt = ffmpeg_next::Packet::empty(); - while v_enc.receive_packet(&mut pkt).is_ok() { - pkt.set_stream(0); - pkt.rescale_ts(v_enc.time_base(), out_v.time_base()); - pkt.write_interleaved(&mut octx).ok(); - } - } - v_enc.send_eof().ok(); - let mut pkt = ffmpeg_next::Packet::empty(); - while v_enc.receive_packet(&mut pkt).is_ok() { - pkt.set_stream(0); - pkt.rescale_ts(v_enc.time_base(), out_v.time_base()); - pkt.write_interleaved(&mut octx).ok(); - } - - // ── Flush audio encoder ─────────────────────────────────────────────── - if let (Some(ref mut a_dec), Some(ref mut a_enc), Some(out_a)) = - (a_dec_opt, a_enc_opt, out_a_idx) - { - a_dec.send_eof().ok(); - let mut af = frame::Audio::empty(); - while a_dec.receive_frame(&mut af).is_ok() { - a_enc.send_frame(Some(&af)).ok(); - } - a_enc.send_eof().ok(); - let mut pkt = ffmpeg_next::Packet::empty(); - while a_enc.receive_packet(&mut pkt).is_ok() { - pkt.set_stream(out_a); - pkt.write_interleaved(&mut octx).ok(); - } - } - - octx.write_trailer() - .context("ffmpeg_transcode_to_webm: write_trailer")?; - - Ok(()) -} - -// ─── Audio waveform PNG ─────────────────────────────────────────────────────── - -/// Render a waveform image for an audio file via libavfilter's `showwavespic`. -/// -/// Equivalent to: -/// ffmpeg -i -/// -filter_complex "showwavespic=s=WxH:colors=0x888888" -/// -frames:v 1 -/// -/// Called from `workers/mod.rs` inside `spawn_blocking`. -/// -/// # Errors -/// Returns an error if the filtergraph cannot be built or the PNG cannot be -/// written. -pub fn ffmpeg_audio_waveform( - input: &Path, - output: &Path, - width: u32, - height: u32, -) -> Result<()> { - init_ffmpeg(); - - // ── Build filter graph ──────────────────────────────────────────────── - // showwavespic reads the *entire* audio stream and produces a single - // frame. We route it through a buffer source → showwavespic → buffersink. - let mut ictx = format::input(input) - .with_context(|| format!("ffmpeg_audio_waveform: cannot open {}", input.display()))?; - - let in_stream = ictx - .streams() - .best(media::Type::Audio) - .context("ffmpeg_audio_waveform: no audio stream")?; - let in_idx = in_stream.index(); - - let mut a_dec = codec::context::Context::from_parameters(in_stream.parameters()) - .context("ffmpeg_audio_waveform: decoder context")? - .decoder() - .audio() - .context("ffmpeg_audio_waveform: open decoder")?; - - // Build lavfi graph: - // abuffer → showwavespic=s=WxH:colors=0x888888 → buffersink - let filter_str = format!("showwavespic=s={width}x{height}:colors=0x888888"); - - let mut graph = filter::Graph::new(); - - // abuffer: feed raw audio frames into the graph. - let abuf_args = format!( - "sample_rate={}:sample_fmt={}:channel_layout=0x{:x}:time_base={}/{}", - a_dec.rate(), - ffmpeg_next::format::Sample::name(a_dec.format()), - a_dec.channel_layout().bits(), - in_stream.time_base().0, - in_stream.time_base().1, - ); - graph - .add(&filter::find("abuffer").context("abuffer filter not found")?, "in", &abuf_args) - .context("ffmpeg_audio_waveform: add abuffer")?; - - // showwavespic filter - graph - .add( - &filter::find("showwavespic").context("showwavespic filter not found")?, - "showwavespic", - &filter_str, - ) - .context("ffmpeg_audio_waveform: add showwavespic")?; - - // buffersink: pull the rendered frame out. - graph - .add( - &filter::find("buffersink").context("buffersink filter not found")?, - "out", - "", - ) - .context("ffmpeg_audio_waveform: add buffersink")?; - - // Link: in → showwavespic → out - { - let mut in_node = graph.get("in").context("ffmpeg_audio_waveform: get abuffer")?; - let mut wave_node = graph.get("showwavespic").context("ffmpeg_audio_waveform: get showwavespic")?; - let mut out_node = graph.get("out").context("ffmpeg_audio_waveform: get buffersink")?; - in_node - .output("default", 0) - .context("ffmpeg_audio_waveform: abuffer output")? - .input("default", 0) - .context("ffmpeg_audio_waveform: showwavespic input")? - .add() - .context("ffmpeg_audio_waveform: link in→wave")?; - wave_node - .output("default", 0) - .context("ffmpeg_audio_waveform: showwavespic output")? - .input("default", 0) - .context("ffmpeg_audio_waveform: buffersink input")? - .add() - .context("ffmpeg_audio_waveform: link wave→out")?; - } - graph - .validate() - .context("ffmpeg_audio_waveform: filter graph validate")?; - - // ── Feed all audio frames into the graph ────────────────────────────── - for (stream, packet) in ictx.packets() { - if stream.index() != in_idx { - continue; - } - a_dec - .send_packet(&packet) - .context("ffmpeg_audio_waveform: send_packet")?; - let mut frame = frame::Audio::empty(); - while a_dec.receive_frame(&mut frame).is_ok() { - graph - .get("in") - .context("ffmpeg_audio_waveform: get abuffer for push")? - .source() - .add(&frame.into()) - .context("ffmpeg_audio_waveform: push frame")?; - } - } - // Signal EOF so showwavespic renders the final frame. - a_dec.send_eof().ok(); - let mut frame = frame::Audio::empty(); - while a_dec.receive_frame(&mut frame).is_ok() { - graph - .get("in") - .unwrap() - .source() - .add(&frame.into()) - .ok(); - } - graph - .get("in") - .unwrap() - .source() - .flush() - .context("ffmpeg_audio_waveform: flush abuffer")?; - - // ── Pull the rendered video frame from the sink ─────────────────────── - let mut rendered = frame::Video::empty(); - graph - .get("out") - .context("ffmpeg_audio_waveform: get buffersink for pull")? - .sink() - .frame(&mut rendered) - .context("ffmpeg_audio_waveform: pull rendered frame")?; - - // ── Encode rendered frame as PNG ────────────────────────────────────── - let png_codec = codec::encoder::find_by_name("png") - .context("png encoder not found in static build")?; - - let mut octx = format::output(output) - .with_context(|| format!("ffmpeg_audio_waveform: cannot open output {}", output.display()))?; - - let mut enc_ctx = codec::context::Context::new_with_codec(png_codec) - .encoder() - .video() - .context("ffmpeg_audio_waveform: PNG encoder context")?; - - enc_ctx.set_width(width); - enc_ctx.set_height(height); - enc_ctx.set_format(ffmpeg_next::format::Pixel::RGB24); - enc_ctx.set_time_base(Rational(1, 1)); - - let mut enc = enc_ctx - .open() - .context("ffmpeg_audio_waveform: open PNG encoder")?; - - // showwavespic outputs RGBA; convert to RGB24 for PNG encoder. - let mut rgb = frame::Video::new(ffmpeg_next::format::Pixel::RGB24, width, height); - let mut sws = SwsContext::get( - rendered.format(), width, height, - ffmpeg_next::format::Pixel::RGB24, width, height, - SwsFlags::BILINEAR, - ) - .context("ffmpeg_audio_waveform: sws for RGB24 convert")?; - sws.run(&rendered, &mut rgb) - .context("ffmpeg_audio_waveform: sws_scale to RGB24")?; - rgb.set_pts(Some(0)); - - let mut out_stream = octx.add_stream(png_codec) - .context("ffmpeg_audio_waveform: add PNG stream")?; - out_stream.set_parameters(&enc); - - octx.write_header() - .context("ffmpeg_audio_waveform: write_header")?; - - enc.send_frame(Some(&rgb)) - .context("ffmpeg_audio_waveform: send_frame")?; - enc.send_eof() - .context("ffmpeg_audio_waveform: send_eof")?; - - let mut pkt = ffmpeg_next::Packet::empty(); - while enc.receive_packet(&mut pkt).is_ok() { - pkt.set_stream(0); - pkt.write_interleaved(&mut octx) - .context("ffmpeg_audio_waveform: write_interleaved")?; - } - - octx.write_trailer() - .context("ffmpeg_audio_waveform: write_trailer")?; - - Ok(()) -} - -// ─── Dead-code stubs (kept for API compatibility) ───────────────────────────── - -/// Formerly converted GIF → WebM/VP9. Superseded by animated WebP path. -/// Retained as dead code. Remove in a follow-up cleanup. -#[allow(dead_code)] -pub fn ffmpeg_gif_to_webm(input: &Path, output: &Path) -> Result<()> { - ffmpeg_transcode_to_webm(input, output) -} - -// ─── Internal helpers ───────────────────────────────────────────────────────── - -/// Compute output dimensions that fit within `max_dim × max_dim` while -/// preserving the source aspect ratio. The smaller axis is rounded to an -/// even number (required by many YUV codecs). -fn scale_dims(src_w: u32, src_h: u32, max_dim: u32) -> (u32, u32) { - if src_w == 0 || src_h == 0 { - return (max_dim, max_dim); - } - let (w, h) = if src_w >= src_h { - let h = (src_h * max_dim / src_w).max(2) & !1; - (max_dim, h) - } else { - let w = (src_w * max_dim / src_h).max(2) & !1; - (w, max_dim) - }; - (w, h) -} -``` - ---- - -## 3. `src/workers/mod.rs` — surgical changes only - -Only the two sections that spawn `TokioCommand::new("ffmpeg")` change. -Everything else (prepare, finalise, timeout logic, DB updates) is untouched. - -### 3a. Remove the import - -```rust -// REMOVE this line: -use tokio::process::Command as TokioCommand; -// REMOVE this if only used by ffmpeg spawn: -use std::process::Stdio; -``` - -### 3b. `transcode_video()` — replace the subprocess block - -The function currently has three phases: `transcode_video_prepare` (spawn_blocking), -subprocess spawn + wait, `transcode_video_finalise` (spawn_blocking). -Replace only **phase 2** (lines roughly 490–520): - -```rust -// ── REMOVE: TokioCommand subprocess spawn ───────────────────────────────── -// let child = TokioCommand::new("ffmpeg") -// .args(&args) -// .stderr(Stdio::piped()) -// .stdout(Stdio::null()) -// .kill_on_drop(true) -// .spawn() -// .map_err(|e| anyhow::anyhow!("failed to spawn ffmpeg: {e}"))?; -// -// match timeout(ffmpeg_timeout, child.wait_with_output()).await { ... } - -// ── REPLACE WITH: in-process library call ──────────────────────────────── -// `args` is no longer needed — pass src/dst paths directly. -// Re-derive them from the prepare result (src_path and tmp.path()). -let src_path2 = src_path.clone(); -let tmp_path2 = tmp.path().to_path_buf(); -let timed_out = timeout( - ffmpeg_timeout, - tokio::task::spawn_blocking(move || { - crate::media::ffmpeg::ffmpeg_transcode_to_webm(&src_path2, &tmp_path2) - }), -) -.await; - -match timed_out { - Ok(Ok(Ok(()))) => {} - Ok(Ok(Err(e))) => return Err(e), - Ok(Err(join_err)) => { - return Err(anyhow::anyhow!("spawn_blocking panicked: {join_err}")) - } - Err(_elapsed) => { - // tmp is still alive here — NamedTempFile drops and removes the - // partial output when this function returns, so no cleanup needed. - warn!( - "VideoTranscode: post {post_id} timed out after {timeout_secs}s" - ); - return Err(anyhow::anyhow!( - "transcode timed out after {timeout_secs}s" - )); - } -} -``` - -> **Note on `args`**: `transcode_video_prepare` builds a `Vec` of -> ffmpeg CLI arguments. After migration this vector is unused. You can either -> leave the prepare function as-is (the `args` binding is silently dropped) or -> simplify `transcode_video_prepare` to return only -> `(src_path, webm_abs, webm_rel, webm_name, tmp)` by removing the argument -> construction block. The prepare function is only called from one place, so -> either approach is safe. - -### 3b. `generate_waveform()` — replace the subprocess block - -Same pattern. Replace only the `TokioCommand` phase between `waveform_prepare` -and `waveform_finalise`: - -```rust -// ── REMOVE: TokioCommand subprocess spawn ───────────────────────────────── -// let child = TokioCommand::new("ffmpeg") -// .args(&args) -// .stderr(Stdio::piped()) -// .stdout(Stdio::null()) -// .kill_on_drop(true) -// .spawn() -// .map_err(...)?; -// match timeout(ffmpeg_timeout, child.wait_with_output()).await { ... } - -// ── REPLACE WITH ────────────────────────────────────────────────────────── -let src_path3 = src.clone(); // src comes from waveform_prepare -let out_path3 = png_abs.clone(); -let thumb_size = CONFIG.thumb_size; - -let timed_out = timeout( - ffmpeg_timeout, - tokio::task::spawn_blocking(move || { - crate::media::ffmpeg::ffmpeg_audio_waveform( - &src_path3, - &out_path3, - thumb_size, - thumb_size / 2, - ) - }), -) -.await; - -match timed_out { - Ok(Ok(Ok(()))) => {} - Ok(Ok(Err(e))) => return Err(e), - Ok(Err(join_err)) => { - return Err(anyhow::anyhow!("spawn_blocking panicked: {join_err}")) - } - Err(_elapsed) => { - warn!("AudioWaveform: post {post_id} timed out after {timeout_secs}s"); - return Err(anyhow::anyhow!( - "waveform timed out after {timeout_secs}s" - )); - } -} -``` - -> **Note**: `waveform_prepare` currently passes `src_str` and `tmp_str` as -> strings for the CLI arg list. After migration the only paths needed are the -> original `src: PathBuf` and the temp file path from `tmp_png.path()`. -> The prepare function can be simplified to not build a `Vec` at all, -> but this is optional cleanup. - ---- - -## 4. `src/detect.rs` — ffmpeg section replacement - -Replace the entire ffmpeg detection section (roughly lines 38–175) with: - -```rust -// ─── ffmpeg ─────────────────────────────────────────────────────────────────── - -/// Initialise the statically linked ffmpeg library and report it as available. -/// -/// Previously probed for the `ffmpeg` binary on PATH. Now just calls -/// `init_ffmpeg()` and always returns `Available`. -/// -/// The `require_ffmpeg` parameter is retained for API compatibility; it is -/// ignored because ffmpeg is always present in a static build. -pub fn detect_ffmpeg(_require_ffmpeg: bool) -> ToolStatus { - crate::media::ffmpeg::init_ffmpeg(); - tracing::info!( - target: "detect", - available = true, - "ffmpeg compiled-in — media conversion and thumbnails always enabled" - ); - ToolStatus::Available -} - -/// Always returns true: libwebp is compiled into the static ffmpeg build. -pub fn detect_webp_encoder(_ffmpeg_ok: bool) -> bool { - tracing::info!(target: "detect", webp = true, "libwebp compiled-in"); - true -} - -/// Always returns true: libvpx-vp9 and libopus are compiled in. -pub fn detect_webm_encoder(_ffmpeg_ok: bool) -> bool { - tracing::info!( - target: "detect", - vp9 = true, - opus = true, - "VP9 + Opus compiled-in — MP4→WebM transcoding always enabled" - ); - true -} -``` - -Delete the following functions entirely (they are only reached when a codec is -absent, which can never happen with a static build): - -- `webp_install_hint()` -- `webm_install_hint(has_vp9, has_opus)` - ---- - -## 5. Optional: simplify `src/media/mod.rs` - -`MediaProcessor::new()` currently probes for ffmpeg at construction time. -With a static build the probes always succeed, so the constructor can be -simplified. This is safe to skip — the existing code still works correctly. - -```rust -// Optional simplification of MediaProcessor::new() -#[must_use] -pub fn new() -> Self { - // ffmpeg is compiled in — no runtime probe needed. - crate::media::ffmpeg::init_ffmpeg(); - Self { - ffmpeg_available: true, - ffmpeg_webp_available: true, - } -} -``` - ---- - -## 6. Summary of changed files - -| File | Change | -|---|---| -| `Cargo.toml` | Add `ffmpeg-next` with `static` feature | -| `.cargo/config.toml` | New file — pin `FFMPEG_BUILD_VERSION` | -| `src/media/ffmpeg.rs` | Complete replacement (see §2) | -| `src/workers/mod.rs` | Replace two `TokioCommand` blocks (see §3) | -| `src/detect.rs` | Replace ffmpeg detection section (see §4) | -| `src/media/mod.rs` | Optional simplification of `new()` (see §5) | -| `src/media/convert.rs` | **No changes** | -| `src/media/thumbnail.rs` | **No changes** | -| `src/server/server.rs` | **No changes** | -| `src/middleware/mod.rs` | **No changes** | From 7aa36c9a93b2e06744f7f8fedaa9ae0ead0bf672 Mon Sep 17 00:00:00 2001 From: csd113 Date: Sun, 22 Mar 2026 18:44:04 -0700 Subject: [PATCH 2/2] bug fixes for arti implementation --- .gitignore | 2 + CHANGELOG.md | 300 +++++++++++++++++-------------- Cargo.lock | 23 ++- Cargo.toml | 11 +- src/config.rs | 144 ++++++++++++++- src/db/mod.rs | 92 ++++++++-- src/detect.rs | 403 +++++++++++++++++++++++++++++++++++------- src/logging.rs | 20 ++- src/middleware/mod.rs | 46 ++++- src/server/server.rs | 105 +++++++++-- 10 files changed, 904 insertions(+), 242 deletions(-) diff --git a/.gitignore b/.gitignore index a86d7d7..4a62330 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ target # These are backup files generated by rustfmt **/*.rs.bk /clippy_reports +build-release.sh +tor_audit_report.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b5f9e2..f94f8b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,235 +6,271 @@ All notable changes to RustChan will be documented in this file. ## [1.1.0 alpha 2] -### Fixed +The headline change in this release is a deep security and correctness audit of the Arti/Tor implementation introduced in alpha 1, resulting in six critical fixes, nine high-priority fixes, and a set of new operator-facing configuration options. Alongside that, this release includes reliability improvements to shutdown coordination, backup handling, multipart parsing, and the database layer. -#### 🔴 Critical — HTTP 500 errors on pages with gateway posts -**Problem:** Posts from the ChanNet gateway have no IP address, causing crashes when pages try to display them. +--- + +### 🔒 Tor / Arti — Security & Correctness Audit -**Solution:** Changed `ip_hash` from `String` to `Option` throughout the codebase so `NULL` values are handled gracefully instead of panicking. +#### Architecture -**Files changed:** -- `src/models.rs`, `src/db/posts.rs`, `src/db/admin.rs` — Handle optional IP hashes -- `src/templates/thread.rs`, `src/templates/admin.rs` — Render empty string for missing IPs -- `src/handlers/admin/backup.rs`, `src/handlers/backup.rs` — Preserve `NULL` on backup/restore +The hidden service implementation from alpha 1 has been audited and corrected. The core architecture — bootstrapping Arti in-process, deriving a `.onion` address from a persistent Ed25519 keypair, and proxying inbound onion streams to the local HTTP port — is unchanged. What changed is correctness, isolation, and operational safety. --- -#### 🟠 Log files written to wrong directory -**Problem:** Logs were created in the executable folder instead of `rustchan-data/`. +#### 🔴 Critical fixes + +**Per-stream IP isolation for Tor users** -**Solution:** Pass `data_dir` instead of `binary_dir` to the logger, and create the directory *before* logging initializes. +Previously every Tor user resolved to `127.0.0.1` as their client IP. The Arti proxy was a raw TCP passthrough (`copy_bidirectional`) with no HTTP awareness, so no header injection was possible. This meant all Tor users shared a single rate-limit bucket, ban entry, and post cooldown: banning one Tor user banned everyone on Tor simultaneously. -**Files changed:** `src/main.rs` +Fixed by introducing `TOR_STREAM_TOKENS`, a `DashMap>` in `detect.rs` keyed by the ephemeral local port of each proxy connection. When `proxy_tor_stream` connects to the local axum socket, the OS assigns an ephemeral source port; axum's `ConnectInfo` sees this as the peer port on the accepted socket. A random `tor:` token is inserted into the map under that port, and a `TokenGuard` RAII struct removes it when the task ends. Both `ClientIp::from_request_parts` and `extract_ip` now look up the peer port in `TOR_STREAM_TOKENS` when the connection is from loopback with `enable_tor_support=true`, returning the per-stream token instead of `127.0.0.1`. Every Tor stream now has its own isolated bucket for rate limiting, bans, and post cooldowns. + +**Files:** `src/detect.rs`, `src/middleware/mod.rs` --- -#### 🟠 Log file names have wrong extension -**Problem:** Rotated logs named `rustchan.log.2024-01-15` instead of `rustchan.2024-01-15.log`. +**Tor-only mode (`tor_only` setting)** + +With `enable_tor_support = true` and the default `bind_addr = 0.0.0.0:8080`, the HTTP server was reachable directly over clearnet simultaneously with the hidden service. An operator expecting a private Tor-only site had no way to enforce that without manually overriding `bind_addr`. -**Solution:** Use `RollingFileAppender::builder()` with `.filename_prefix()` and `.filename_suffix()` for correct naming. +Added a new `tor_only` setting to `settings.toml`. When `tor_only = true` and `enable_tor_support = true`, `bind_addr` is silently overridden to `127.0.0.1:{port}` during config loading — the port is preserved, only the host changes. The override is logged at startup. Default remains `false` (dual-stack: clearnet and Tor both active), which is the correct default for an imageboard that wants to be reachable both ways. -**Files changed:** `src/logging.rs` +```toml +# Restrict to Tor-only (hidden service). Clearnet access blocked. +# tor_only = false +``` + +**Files:** `src/config.rs` --- -#### 🟡 Log files changed from JSON to readable text -**Problem:** Logs were dense JSON, hard to read with `tail`, `grep`, etc. +**Graceful shutdown for the Tor task** + +The Tor retry loop had no `CancellationToken`. During shutdown, `worker_cancel.cancel()` signaled every other background task but the Tor task continued running — sleeping through a backoff of up to 480 seconds. The shutdown code then hit a hard 10-second timeout and abandoned the task, leaving Tor circuits open without sending `RELAY_END` cells. +Fixed by adding a `cancel: CancellationToken` parameter to `detect_tor()`. Both the `run_arti(...)` call and the backoff sleep now use `tokio::select!` against the token, so the task exits promptly when shutdown is signaled. The `worker_cancel` variable in `run_server()` is moved to before the `detect_tor` call so it is available to pass in. The shutdown timeout is extended from 10s to 15s as a safety net for any in-flight `copy_bidirectional` draining — in practice the task exits in milliseconds once the token fires. -## Reliability & Shutdown Improvements +**Files:** `src/detect.rs`, `src/server/server.rs` --- -## Multipart Handling & Memory Safety (`src/handlers/mod.rs`) +**`tor_client` and `onion_service` explicit keepalive** -- Added strict per-field size limits for multipart text inputs: - - Post body capped (~100KB) - - Name, subject, and other fields capped (~4KB) -- Replaced unbounded `field.text()` usage with controlled byte-reading logic -- Prevented large heap allocations from oversized text fields -- Eliminated OOM risk under concurrent large-form submissions +`tor_client` is last used on the line that calls `launch_onion_service`. `onion_service` is last used inside the `HsId` retry block. Both have side-effectful `Drop` implementations: dropping `tor_client` closes all Tor circuits; dropping `onion_service` deregisters the hidden service from the Tor network. Both variables must stay alive through the entire stream loop. + +Rust named `let` bindings drop at end of their enclosing scope (the function body), not at last-use, so this was not a live bug — but it was invisible and fragile. Added explicit `let _ = &tor_client; let _ = &onion_service;` keepalive borrows at the end of `run_arti`, after the stream loop exits, making the intent unambiguous and guarding against any future tooling that might warn about "unused" bindings. + +**Files:** `src/detect.rs` -- Hardened poll duration parsing: - - Added validation before multiplication - - Prevented intermediate integer overflow prior to clamping +--- + +#### 🟠 High-priority fixes + +**Onion address encoder: fixed checksum computation** + +In `hsid_to_onion_address`, the two checksum bytes were extracted from the `Sha3_256` digest using an iterator with `.unwrap_or(0)` fallbacks. `Sha3_256` always produces 32 bytes so the fallback was dead code, but it masked the logic and would silently produce a wrong checksum if the digest size ever changed. Replaced with direct array indexing: `let hash: [u8; 32] = hasher.finalize().into(); let checksum = [hash[0], hash[1]];`. + +Added a Python-verified cryptographic test vector for the all-zeros Ed25519 key: +``` +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaam2dqd.onion +``` +Verified with: +```python +import hashlib, base64 +pub = bytes(32); ver = bytes([3]) +chk = hashlib.sha3_256(b'.onion checksum' + pub + ver).digest()[:2] +print(base64.b32encode(pub+chk+ver).decode().lower().rstrip('=')+'.onion') +``` + +**Files:** `src/detect.rs` --- -## Backup System Reliability (`src/handlers/admin/backup.rs`) +**`Onion-Location` response header for Tor Browser** -- Replaced fragile `VACUUM INTO` string-based SQL with `rusqlite::backup` API -- Eliminated dependency on manual SQL escaping and path formatting -- Improved cross-platform correctness and error transparency +Tor Browser reads the `Onion-Location` response header and automatically prompts the user to switch to the `.onion` address when browsing the clearnet version of a site. The header was never set anywhere in the codebase. -- Implemented guaranteed temporary file cleanup: - - Introduced RAII-style cleanup mechanism - - Ensures backup artifacts are removed even on: - - client disconnects - - early termination - - runtime drops +Added `onion_location_middleware` — an async middleware function that reads `state.onion_address`, and when the address is known and the response `Content-Type` is `text/html`, inserts `Onion-Location: http://` into the response headers. Wired into `build_router` via `axum_middleware::from_fn_with_state` at the outermost position so it fires on every HTML response. Non-HTML responses (static assets, JSON, media) are skipped. -- Improved error signaling: - - Database pool exhaustion now correctly returns retryable errors (503) instead of 500 +**Files:** `src/server/server.rs` --- -## Tor — Arti In-Process Migration +**Configurable bootstrap timeout** -Replaced the subprocess-based C Tor launcher with **[Arti](https://gitlab.torproject.org/tpo/core/arti)** running fully in-process. **No system `tor` installation is required.** +The Tor bootstrap timeout was hardcoded at 120 seconds. On censored networks using bridges or pluggable transports, directory fetch is slow and 120 seconds is insufficient — the task would time out, wait through exponential backoff, and retry indefinitely without ever succeeding. -**How it works:** at startup a single Tokio task bootstraps Arti, derives a `.onion` address from a persistent Ed25519 keypair, launches the hidden service, and proxies inbound onion connections to the local HTTP port — all without spawning a child process, writing a `torrc`, or polling a `hostname` file. +Added `tor_bootstrap_timeout_secs` to `settings.toml` (default 120). The timeout error message now includes a hint to increase this value. -- Bootstrap takes ~30 s on first run (Arti downloads ~2 MB of directory data) and ~5 s on subsequent runs (consensus cached in `arti_cache/`) -- The onion address is published to `AppState` the moment the service is ready; handlers read it from memory with zero filesystem I/O per request -- Service keypair lives in `rustchan-data/arti_state/keys/` — back this directory up to preserve your `.onion` address; delete it to rotate to a new one -- Old `rustchan-data/tor_data/`, `rustchan-data/tor_hidden_service/`, and `rustchan-data/torrc` are no longer created and can be safely deleted after migration -- **Note:** the keypair location changed on migration (`tor_hidden_service/` → `arti_state/keys/`), so a new `.onion` address is generated on first run unless the old Ed25519 key is manually imported via Arti's key management tooling +```toml +# Increase for censored networks or when using bridges. +# tor_bootstrap_timeout_secs = 120 +``` -**Files changed:** `Cargo.toml` (+6 deps: `arti-client`, `tor-hsservice`, `tor-cell`, `futures`, `sha3`, `data-encoding`), `src/detect.rs`, `src/middleware/mod.rs`, `src/server/server.rs`, `src/handlers/board.rs`, `src/handlers/admin/mod.rs`, `src/handlers/admin/settings.rs`, `src/config.rs` +**Files:** `src/config.rs`, `src/detect.rs` --- -## Database Layer Improvements (`src/db/mod.rs`) +**Configurable maximum concurrent Tor streams** + +`MAX_CONCURRENT_TOR_STREAMS` was a hardcoded compile-time constant (`512`). Operators on resource-constrained hosts (low FD limits, limited RAM) had no way to reduce it without recompiling. -- Made database connection pool size configurable via environment/config -- Removed hardcoded pool limit to improve scalability under load +Added `tor_max_concurrent_streams` to `settings.toml` (default 512). When the limit is reached, `stream_req` is dropped explicitly — Arti sends a `RELAY_END` cell automatically on drop. -- Corrected error mapping: - - `r2d2::Error` (pool exhaustion) now maps to `503 Service Unavailable` - - Prevents misclassification of load-related failures as internal errors +```toml +# Reduce if the process hits file descriptor limits. +# tor_max_concurrent_streams = 512 +``` -- Removed silent fallback in initialization logic: - - Replaced `unwrap_or(0)` with proper error propagation - - Prevents incorrect “first-run” detection on DB failure +**Files:** `src/config.rs`, `src/detect.rs` --- -## Transaction Safety & Concurrency (`src/db/threads.rs`, `src/db/posts.rs`, `src/db/admin.rs`, `src/db/boards.rs`) +**Infrastructure errors distinguished from normal stream closure** + +All errors from `proxy_tor_stream` were logged at `DEBUG` with the message "Tor: stream closed". This made it impossible to distinguish a normal client disconnect (expected, routine) from "local TCP connect failed" (axum has crashed or is unrestarted — requires operator attention). + +Split error handling: connection failures to the local HTTP server now log at `ERROR` with a clear message ("Tor: cannot reach local HTTP server — is axum running?"). Normal stream closures (EOF, client disconnect, keep-alive expiry) continue to log at `DEBUG`. -- Replaced `unchecked_transaction()` (DEFERRED) with explicit `BEGIN IMMEDIATE` -- Ensured write locks are acquired at transaction start -- Eliminated mid-transaction lock upgrade failures (`SQLITE_BUSY`) -- Improved consistency and reliability under concurrent write load +**Files:** `src/detect.rs` --- -## Logging System Stability (`src/logging.rs`) +**Attempt counter reset after healthy session** -- Replaced unbounded log file (`rolling::never`) with rotating log strategy -- Prevented uncontrolled log growth and disk exhaustion -- Improved long-term operational stability in production environments +The exponential backoff retry counter (`attempt`) incremented on both crash exits and clean exits. After 4 clean reconnect cycles, the service was waiting 480 seconds between restart attempts — identical behavior to a crash loop. A clean exit after ≥60 seconds of healthy operation now resets `attempt = 0`. + +**Files:** `src/detect.rs` --- -## HTTP Response Correctness (`src/handlers/thread.rs`, `src/handlers/board.rs`) +**`Arc` for the local address string** + +`local_addr` was a `String` cloned into every spawned proxy task — one heap allocation per Tor connection. Replaced with `Arc`, making each clone an atomic reference count increment with no heap allocation. -- Removed `.unwrap_or_default()` from 304 response builders -- Replaced with explicit, safe response construction -- Ensured correct HTTP semantics for cache validation responses +**Files:** `src/detect.rs` --- -## Configuration File Safety (`src/config.rs`) +**Configurable service nickname** + +The Arti onion service nickname was hardcoded to `"rustchan"`. When multiple instances share the same `arti_state/` directory (e.g. Docker volume mounts, CI), identical nicknames cause key collisions and one instance fails to start its onion service. -- Replaced non-atomic file writes with atomic write pattern: - - write to temporary file - - persist via rename -- Prevented configuration corruption on crash or partial write +Added `tor_service_nickname` to `settings.toml` (default `"rustchan"`). + +```toml +# Change when running multiple instances sharing the same arti_state/ directory. +# tor_service_nickname = "rustchan" +``` + +**Files:** `src/config.rs`, `src/detect.rs` --- -## Cross-Cutting Improvements +**Onion address omitted from structured INFO log** + +The onion address was logged as a structured field at `INFO` level, causing it to appear in plaintext in the JSON log file (`rustchan.log`) and any log aggregator or forwarding pipeline it feeds into. For operators running a sensitive hidden service, this is unwanted metadata exposure. + +The address is now logged as a structured field only at `DEBUG`. A bare `INFO` event ("Tor: hidden service active") fires without the address. The TTY banner and admin panel always show the full address. -### Error Handling +**Files:** `src/detect.rs` -- Reduced silent error suppression patterns -- Improved propagation and visibility of operational failures -- Increased observability of system state under failure conditions +--- + +#### 🟡 Other Arti changes + +- **`yield_now()` in rendezvous loop** — `stream_requests.next().await` runs in a tight async loop. Under a connection flood, the task could monopolize the Tokio executor thread between `next()` returning and the `tokio::spawn` call. Added `tokio::task::yield_now().await` at the top of the loop body. +- **Local connect timeout increased** — the timeout for `proxy_tor_stream` to connect to the local axum socket was 5 seconds. Under load the axum TCP accept queue fills and `connect()` can legitimately take longer. Increased to 15 seconds. +- **Dead `ToolStatus::Spawning` variant removed** — `Spawning` was added for the old subprocess-based Tor launcher. `detect_tor` now returns `Option>` and never produces this variant. Removed to prevent future code from adding unreachable match arms. +- **`stream_req.target()` call removed** — this method does not exist on `StreamRequest` in `tor-hsservice 0.40`. The diagnostic log line it produced has been removed. +- **`run_arti` refactored for line-count compliance** — the onion address publication and TTY banner block extracted into `async fn publish_onion_address()`, keeping `run_arti` under the clippy line-count threshold. --- -### Database Reliability +### 🔴 Critical — HTTP 500 errors on pages with gateway posts -- Standardized transaction patterns across modules -- Improved behavior under high contention scenarios -- Reduced retry loops and transient DB errors +Posts inserted via the ChanNet gateway carry no IP address. Pages that attempted to display or process these posts were crashing because `ip_hash` was typed as `String` and the `NULL` database value caused a panic. + +Changed `ip_hash` to `Option` throughout. `NULL` values are now handled gracefully — they render as an empty string in templates and are passed through backup/restore without modification. + +**Files:** `src/models.rs`, `src/db/posts.rs`, `src/db/admin.rs`, `src/templates/thread.rs`, `src/templates/admin.rs`, `src/handlers/admin/backup.rs`, `src/handlers/backup.rs` --- -## Summary +### 🟠 Reliability & Shutdown + +**Worker lifecycle** -These changes collectively improve: +- Persisted `JoinHandle`s returned by the worker pool; shutdown now awaits each worker with a bounded per-worker timeout instead of a blind fixed sleep +- Signaling via `CancellationToken` threaded through every worker task +- Prevents corruption of in-progress FFmpeg transcodes during shutdown +- Added startup recovery to reset jobs stuck in `running` state after an unclean exit -- Memory safety under user input -- Database correctness under concurrency -- Crash resilience and panic handling -- Backup integrity and filesystem safety -- Logging reliability and disk usage control -- Accuracy of error reporting and HTTP responses +**ChanNet server** -Resulting in a significantly more robust and production-ready system. +- Added graceful shutdown support to the ChanNet listener (port 7070) +- Unified shutdown signal with the main HTTP server so in-flight federation requests drain before the process exits -### Worker Lifecycle Management (`src/server/server.rs`, `src/workers/mod.rs`) +**Background tasks** -- Persisted `JoinHandle`s returned by the worker pool instead of discarding them -- Implemented proper graceful shutdown by: - - Signaling worker cancellation via `CancellationToken` - - Awaiting all worker tasks with bounded timeouts -- Eliminated reliance on fixed sleep-based shutdown timing -- Prevented corruption of in-progress jobs (e.g., FFmpeg transcodes) -- Enabled deterministic shutdown behavior for background workers +- All periodic background tasks (session purge, WAL checkpoint, IP prune, login-fail prune, VACUUM, poll cleanup, cache eviction) replaced infinite loops with `tokio::select!` against the worker cancel token +- Ensures every task exits cleanly on shutdown with no orphaned async tasks -### Job Recovery +**HTTP** -- Added startup recovery logic to reset jobs stuck in `running` state -- Ensures jobs interrupted during shutdown are retried instead of permanently stalled +- Added request timeout middleware (30 seconds) to protect against slow-loris style attacks and stalled client connections --- -## ChanNet Server Shutdown (`src/server/server.rs`) +### 🟠 Multipart Handling -- Added graceful shutdown support to ChanNet server -- Unified shutdown signal with main HTTP server -- Prevents abrupt termination of in-flight federation requests -- Eliminates risk of partial/corrupt response streams during shutdown +- Added strict per-field size limits: post body capped at ~100 KB, name/subject/other fields at ~4 KB +- Replaced unbounded `field.text()` calls with controlled byte-reading that returns `413` the moment the running total exceeds the limit +- Eliminated OOM risk under concurrent large-form submissions +- Hardened poll duration parsing: added overflow validation before the seconds multiplication step + +**Files:** `src/handlers/mod.rs` --- -## Background Task Control (`src/server/server.rs`) +### 🟠 Backup System + +- Replaced `VACUUM INTO` string-based SQL with the `rusqlite::backup` API — eliminates manual SQL escaping and improves cross-platform correctness +- Introduced RAII-style temporary file cleanup: backup artifacts are removed even on client disconnect, early termination, or runtime drop +- Database pool exhaustion during backup now returns 503 (retryable) instead of 500 -- Integrated cancellation awareness into background tasks -- Replaced infinite loops with `tokio::select!` to listen for shutdown signals -- Ensures all periodic tasks (cleanup, pruning, etc.) terminate cleanly -- Reduces risk of abrupt termination mid-operation +**Files:** `src/handlers/admin/backup.rs` --- -## HTTP Reliability (`src/server/server.rs`) +### 🟡 Database -- Added request timeout middleware -- Protects against slow or stalled clients holding connections indefinitely -- Improves resilience against slowloris-style behavior +- Made connection pool size configurable; removed hardcoded pool limit +- `r2d2::Error` (pool exhaustion) now maps to `503 Service Unavailable` instead of 500 +- Removed `unwrap_or(0)` silent fallback in DB initialization — replaced with proper error propagation +- Replaced `unchecked_transaction()` (DEFERRED) with `BEGIN IMMEDIATE` across `threads.rs`, `posts.rs`, `admin.rs`, `boards.rs` — eliminates mid-transaction lock upgrade failures under concurrent write load + +**Files:** `src/db/mod.rs`, `src/db/threads.rs`, `src/db/posts.rs`, `src/db/admin.rs`, `src/db/boards.rs` --- -## Worker System Stability (`src/workers/mod.rs`) +### 🟡 Logging + +- Replaced rolling-never log strategy with a rotating appender to prevent unbounded disk growth +- Fixed log directory: logs now write to `rustchan-data/` instead of the executable folder +- Fixed log filename format: rotated files now named `rustchan.2024-01-15.log` instead of `rustchan.log.2024-01-15` -- Ensured worker pool properly integrates with shutdown lifecycle -- Improved coordination between job queue and worker threads -- Reinforced guarantees around job completion and cancellation handling +**Files:** `src/main.rs`, `src/logging.rs` --- -## Summary +### 🟡 Other fixes -These changes significantly improve: +- **HTTP 304 responses** — removed `.unwrap_or_default()` from 304 response builders; replaced with explicit safe construction +- **Configuration file writes** — replaced non-atomic file writes with write-to-temp-then-rename pattern; prevents `settings.toml` corruption on crash -- Graceful shutdown correctness -- Background task reliability -- Job processing integrity -- Resistance to partial writes and corruption -- Operational stability under restart conditions +**Files:** `src/handlers/thread.rs`, `src/handlers/board.rs`, `src/config.rs` --- @@ -756,4 +792,4 @@ Both handlers now accept `HeaderMap`, derive an ETag (board index: `"{max_bump_t - Rate limiting and CSRF protection - Configurable via `settings.toml` or environment variables - SQLite database with connection pooling -- Nginx and systemd deployment configs included \ No newline at end of file +- Nginx and systemd deployment configs included diff --git a/Cargo.lock b/Cargo.lock index 7435085..6d5843f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1754,6 +1754,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "futures-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" +dependencies = [ + "futures-io", + "rustls", + "rustls-pki-types", +] + [[package]] name = "futures-sink" version = "0.3.32" @@ -3765,7 +3776,7 @@ dependencies = [ [[package]] name = "rustchan" -version = "1.1.0" +version = "1.1.0-alpha.2" dependencies = [ "anyhow", "argon2", @@ -3789,6 +3800,7 @@ dependencies = [ "regex", "reqwest", "rusqlite", + "rustls-webpki", "serde", "serde_json", "sha2", @@ -3839,6 +3851,7 @@ version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -3859,9 +3872,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ "ring", "rustls-pki-types", @@ -5779,11 +5792,15 @@ dependencies = [ "dyn-clone", "educe", "futures", + "futures-rustls", "hex", "libc", "native-tls", "paste", "pin-project", + "rustls", + "rustls-pki-types", + "rustls-webpki", "socket2", "thiserror 2.0.18", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 585f8c0..a25983a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustchan" -version = "1.1.0" +version = "1.1.0-alpha.2" edition = "2021" # axum 0.8 requires Rust 1.75; that is the effective floor for this project. rust-version = "1.90" @@ -85,6 +85,9 @@ zip = { version = "8", default-features = false, features = ["deflate"] } # Step 1.1: reqwest for federation push (chan_refresh) and pull (chan_poll). # rustls-tls avoids a native OpenSSL dependency — single static binary stays intact. reqwest = { version = "0.12", default-features = false, features = ["multipart", "json", "rustls-tls"] } +# Pin to patched version — fixes RUSTSEC-2026-0049 (CRL distribution point +# matching bug). Transitive dep via reqwest/arti-client. +rustls-webpki = "0.103.10" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt", "json"] } @@ -94,7 +97,11 @@ thiserror = "2" tempfile = "3" libc = "0.2" -arti-client = { version = "0.40", features = ["tokio", "native-tls", "onion-service-service"] } +# F-09: Use rustls instead of native-tls to keep the binary self-contained. +# native-tls pulls in OpenSSL as a transitive dep, causing a dynamic link +# against libssl/libcrypto — contradicting the rustls-tls choice on reqwest. +# Verify post-change: cargo tree --edges features | grep openssl (expect nothing) +arti-client = { version = "0.40", features = ["tokio", "rustls", "onion-service-service"] } tor-hsservice = { version = "0.40" } tor-cell = { version = "0.40" } futures = "0.3" diff --git a/src/config.rs b/src/config.rs index ed2cbc7..dffba77 100644 --- a/src/config.rs +++ b/src/config.rs @@ -56,6 +56,25 @@ struct SettingsFile { max_audio_size_mb: Option, cookie_secret: Option, enable_tor_support: Option, + /// When true, the HTTP server binds exclusively to 127.0.0.1 so it is + /// reachable only through the Tor hidden service. Overrides the host + /// portion of `bind_addr` (the configured port is preserved). + /// Default: false (clearnet and Tor both active when `enable_tor_support=true`). + tor_only: Option, + /// Seconds to wait for Tor bootstrap before timing out and retrying. + /// Increase to 300+ on heavily censored networks or when using bridges. + /// Default: 120. + tor_bootstrap_timeout_secs: Option, + /// Maximum simultaneous inbound Tor streams (proxy tasks). + /// Each stream holds one file descriptor. Reduce if the process runs low + /// on FDs; excess connections are dropped with a `RELAY_END` cell. + /// Default: 512. + tor_max_concurrent_streams: Option, + /// Nickname for the Arti onion service key. + /// Must be unique per `arti_state/` directory. Change this when running + /// multiple instances that share the same storage to avoid key collisions. + /// Default: "rustchan". + tor_service_nickname: Option, require_ffmpeg: Option, /// How often to run PRAGMA `wal_checkpoint(TRUNCATE)`, in seconds. /// Set to 0 to disable. Default: 3600 (hourly). @@ -131,7 +150,20 @@ pub fn generate_settings_file_if_missing() { OsRng.fill_bytes(&mut secret_bytes); let secret = hex::encode(secret_bytes); - let content = format!( + let content = settings_template(&secret); + + match std::fs::write(&path, content) { + Ok(()) => println!("Created settings.toml ({})", path.display()), + Err(e) => eprintln!("Warning: could not write settings.toml: {e}"), + } +} + +/// Build the default settings.toml content with the given generated secret. +/// +/// Extracted from `generate_settings_file_if_missing` to keep that function +/// under the line-count lint threshold. +fn settings_template(secret: &str) -> String { + format!( r#"# RustChan — Instance Settings # Edit this file to configure your imageboard. # Restart the server after making changes. @@ -168,6 +200,27 @@ max_audio_size_mb = 150 # Delete that directory to rotate to a new .onion address. enable_tor_support = true +# When true, the HTTP server binds exclusively to 127.0.0.1 so the site is +# reachable ONLY through the Tor hidden service — clearnet access is blocked. +# Requires enable_tor_support = true. Default: false (dual-stack: both +# clearnet and Tor are active simultaneously). +# tor_only = false + +# Seconds to wait for Tor to connect to the network before giving up and +# retrying. The default (120 s) works on open networks. On censored networks +# or when using bridges, increase this to 300 or more. +# tor_bootstrap_timeout_secs = 120 + +# Maximum number of simultaneous inbound Tor connections. +# Each connection holds one file descriptor. Reduce if you hit FD limits. +# tor_max_concurrent_streams = 512 + +# Nickname for this instance's Tor hidden service key. +# Only needs changing when multiple rustchan instances share the same +# rustchan-data/arti_state/ directory — identical nicknames cause key +# collisions and one instance will fail to start its onion service. +# tor_service_nickname = "rustchan" + # Set to true to hard-exit at startup when ffmpeg is not found. # When false (default), the server starts normally and video thumbnails # are replaced with SVG placeholders. @@ -235,12 +288,7 @@ cookie_secret = "{secret}" # Keep on loopback unless RustWave runs on a different host. # chan_net_bind = "127.0.0.1:7070" "# - ); - - match std::fs::write(&path, content) { - Ok(()) => println!("Created settings.toml ({})", path.display()), - Err(e) => eprintln!("Warning: could not write settings.toml: {e}"), - } + ) } // ─── Runtime config ─────────────────────────────────────────────────────────── @@ -265,6 +313,12 @@ pub struct Config { // ── External tool settings ──────────────────────────────────────────────── /// When true, Tor is probed at startup and hints are printed. pub enable_tor_support: bool, + /// Seconds before a bootstrap attempt is considered failed and retried. + pub tor_bootstrap_timeout_secs: u64, + /// Maximum simultaneous inbound Tor proxy tasks. + pub tor_max_concurrent_streams: usize, + /// Nickname for the Arti onion service. Unique per `arti_state/` directory. + pub tor_service_nickname: String, /// When true, the server exits if ffmpeg is missing. pub require_ffmpeg: bool, @@ -357,6 +411,23 @@ impl Config { &format!("{}:{}", env_str("CHAN_HOST", "0.0.0.0"), port), ); + let tor_only = env_bool("CHAN_TOR_ONLY", s.tor_only.unwrap_or(false)); + let enable_tor_support = env_bool("CHAN_TOR_SUPPORT", s.enable_tor_support.unwrap_or(true)); + + // When tor_only=true, force the bind host to 127.0.0.1 regardless of + // what bind_addr or CHAN_HOST are set to. The configured port is kept. + let bind_addr = if tor_only && enable_tor_support { + let port_str = bind_addr.rsplit_once(':').map_or("8080", |(_, p)| p); + tracing::info!( + target: "config", + bind_addr = %format!("127.0.0.1:{port_str}"), + "tor_only=true: overriding bind address to loopback" + ); + format!("127.0.0.1:{port_str}") + } else { + bind_addr + }; + let behind_proxy = env_bool("CHAN_BEHIND_PROXY", false); // Resolve cookie_secret from env > settings.toml. @@ -417,7 +488,19 @@ impl Config { .saturating_mul(1024) .saturating_mul(1024), - enable_tor_support: env_bool("CHAN_TOR_SUPPORT", s.enable_tor_support.unwrap_or(true)), + enable_tor_support, + tor_bootstrap_timeout_secs: env_parse( + "CHAN_TOR_BOOTSTRAP_TIMEOUT", + s.tor_bootstrap_timeout_secs.unwrap_or(120), + ), + tor_max_concurrent_streams: env_parse( + "CHAN_TOR_MAX_STREAMS", + s.tor_max_concurrent_streams.unwrap_or(512), + ), + tor_service_nickname: std::env::var("CHAN_TOR_NICKNAME") + .ok() + .or(s.tor_service_nickname) + .unwrap_or_else(|| "rustchan".to_string()), require_ffmpeg: env_bool("CHAN_REQUIRE_FFMPEG", s.require_ffmpeg.unwrap_or(false)), bind_addr, @@ -559,6 +642,51 @@ impl Config { let _ = std::fs::remove_file(probe); } + // F-13: Pre-flight writability check for Arti data directories. + // Without this, a permissions error on these dirs only surfaces ~30 s + // into bootstrap as a cryptic internal error — invisible at startup. + if self.enable_tor_support { + let exe = std::env::current_exe() + .ok() + .and_then(|p| p.parent().map(std::path::Path::to_path_buf)) + .unwrap_or_else(|| std::path::PathBuf::from(".")); + let data_dir = exe.join("rustchan-data"); + for subdir in ["arti_state", "arti_cache"] { + let dir = data_dir.join(subdir); + std::fs::create_dir_all(&dir).map_err(|e| { + anyhow::anyhow!("CONFIG ERROR: cannot create Tor dir {}: {e}", dir.display()) + })?; + + // Arti requires arti_state/ to have permissions 0700 (no group + // or other read access) for its key material. create_dir_all + // respects the process umask, typically yielding 0755, which + // Arti rejects with "problem with filesystem permissions". + // Explicitly set 0700 on Unix so Arti accepts the directory. + // arti_cache/ holds no sensitive data and is left at normal + // permissions, but we restrict it too for defence-in-depth. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let perms = std::fs::Permissions::from_mode(0o700); + std::fs::set_permissions(&dir, perms).map_err(|e| { + anyhow::anyhow!( + "CONFIG ERROR: cannot set permissions on Tor dir {}: {e}", + dir.display() + ) + })?; + } + + let probe = dir.join(".write_probe"); + std::fs::write(&probe, b"").map_err(|_| { + anyhow::anyhow!( + "CONFIG ERROR: Tor dir {} is not writable — check permissions", + dir.display() + ) + })?; + let _ = std::fs::remove_file(probe); + } + } + // Step 1.2: Validate rustwave_url scheme so operators catch // misconfiguration at startup rather than at first federation call. if !self.rustwave_url.starts_with("http://") && !self.rustwave_url.starts_with("https://") { diff --git a/src/db/mod.rs b/src/db/mod.rs index 6edb0b9..5a2dfec 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -209,22 +209,36 @@ pub fn has_no_admin(pool: &DbPool) -> bool { #[allow(clippy::too_many_lines)] fn create_schema(conn: &rusqlite::Connection) -> Result<()> { + // The highest migration number this binary knows about. Seeded into + // schema_version on fresh installs so all migrations are skipped (the base + // CREATE TABLE statements already include every column they would add). + // Must be updated whenever a new migration entry is appended to the list. + const CURRENT_MAX_MIGRATION: i64 = 26; + // FIX[MED-6]: Use execute_batch for all DDL so it runs in a single // implicit transaction and is idempotent on re-run. conn.execute_batch( " -- Boards table CREATE TABLE IF NOT EXISTS boards ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - short_name TEXT NOT NULL UNIQUE, - name TEXT NOT NULL, - description TEXT NOT NULL DEFAULT '', - nsfw INTEGER NOT NULL DEFAULT 0, - max_threads INTEGER NOT NULL DEFAULT 150, - bump_limit INTEGER NOT NULL DEFAULT 500, - allow_video INTEGER NOT NULL DEFAULT 1, + id INTEGER PRIMARY KEY AUTOINCREMENT, + short_name TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + nsfw INTEGER NOT NULL DEFAULT 0, + max_threads INTEGER NOT NULL DEFAULT 150, + bump_limit INTEGER NOT NULL DEFAULT 500, + allow_video INTEGER NOT NULL DEFAULT 1, allow_tripcodes INTEGER NOT NULL DEFAULT 1, - created_at INTEGER NOT NULL DEFAULT (unixepoch()) + allow_images INTEGER NOT NULL DEFAULT 1, + allow_audio INTEGER NOT NULL DEFAULT 0, + edit_window_secs INTEGER NOT NULL DEFAULT 0, + allow_editing INTEGER NOT NULL DEFAULT 0, + allow_archive INTEGER NOT NULL DEFAULT 1, + allow_video_embeds INTEGER NOT NULL DEFAULT 0, + allow_captcha INTEGER NOT NULL DEFAULT 0, + post_cooldown_secs INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL DEFAULT (unixepoch()) ); -- Threads table (metadata only; OP content is in posts) @@ -301,6 +315,15 @@ fn create_schema(conn: &rusqlite::Connection) -> Result<()> { created_at INTEGER NOT NULL DEFAULT (unixepoch()) ); + -- Ban appeal submissions from banned users + CREATE TABLE IF NOT EXISTS ban_appeals ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ip_hash TEXT NOT NULL, + reason TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'open', + created_at INTEGER NOT NULL DEFAULT (unixepoch()) + ); + -- Word filters CREATE TABLE IF NOT EXISTS word_filters ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -399,24 +422,63 @@ fn create_schema(conn: &rusqlite::Connection) -> Result<()> { ON reports(status, created_at DESC); CREATE INDEX IF NOT EXISTS idx_mod_log_created ON mod_log(created_at DESC); + CREATE INDEX IF NOT EXISTS idx_posts_thread_id + ON posts(thread_id); + CREATE INDEX IF NOT EXISTS idx_posts_ip_hash + ON posts(ip_hash); + CREATE INDEX IF NOT EXISTS idx_threads_archived + ON threads(board_id, archived, bumped_at DESC); + + -- ChanNet federation mirror table (text-only posts from remote nodes) + CREATE TABLE IF NOT EXISTS chan_net_posts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + remote_post_id INTEGER NOT NULL, + board_id INTEGER NOT NULL REFERENCES boards(id) ON DELETE CASCADE, + author TEXT NOT NULL DEFAULT 'anon', + content TEXT NOT NULL DEFAULT '', + remote_ts INTEGER NOT NULL, + imported_at INTEGER NOT NULL DEFAULT (unixepoch()) + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_chan_net_posts_remote + ON chan_net_posts(remote_post_id, board_id); ", ) .context("Schema creation failed")?; // ─── Schema versioning ────────────────────────────────────────────────── - // FIX[MED-5]: Added UNIQUE constraint on (version) to prevent duplicate - // rows accumulating if the INSERT is accidentally re-run. - conn.execute_batch( + // The schema_version table holds exactly one row. + // + // On a fresh install we seed it with the highest known migration number so + // that every migration (whose DDL is already in the CREATE TABLE above) is + // skipped. On an existing DB the INSERT is a no-op (row already present) + // and the stored version is read back unchanged. + // + // `SELECT … WHERE NOT EXISTS` seeds only when the table is empty, avoiding + // the previous bug where `INSERT OR IGNORE … VALUES (0)` would succeed on + // an existing DB (0 ≠ stored_version satisfies the UNIQUE constraint on the + // version column), creating a second row and making the SELECT return an + // unpredictable value — often 0 — causing all migrations to re-run and + // emit spurious "already applied" warnings on every startup. + // + // `MAX(version)` makes the SELECT correct even if a stale second row + // survived from a previous binary version. + conn.execute_batch(&format!( "CREATE TABLE IF NOT EXISTS schema_version ( version INTEGER NOT NULL DEFAULT 0, UNIQUE(version) ); - INSERT OR IGNORE INTO schema_version (version) VALUES (0);", - ) + INSERT INTO schema_version (version) + SELECT {CURRENT_MAX_MIGRATION} + WHERE NOT EXISTS (SELECT 1 FROM schema_version);", + )) .context("Failed to create schema_version table")?; let current_version: i64 = conn - .query_row("SELECT version FROM schema_version", [], |r| r.get(0)) + .query_row( + "SELECT COALESCE(MAX(version), 0) FROM schema_version", + [], + |r| r.get(0), + ) .context("Failed to read schema_version")?; // Each entry is (introduced_at_version, sql). diff --git a/src/detect.rs b/src/detect.rs index 9b809ee..2cfa832 100644 --- a/src/detect.rs +++ b/src/detect.rs @@ -14,8 +14,13 @@ use std::process::{Command, Stdio}; use std::sync::Arc; /// Result of probing for a tool at startup. +/// +/// Note: the `Spawning` variant that existed in v1.0 has been removed. +/// `detect_tor` now returns `Option>` directly; it no longer +/// uses this enum. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ToolStatus { + /// Tool is ready for use immediately (e.g. ffmpeg). Available, Missing, } @@ -32,11 +37,11 @@ pub fn detect_ffmpeg(require_ffmpeg: bool) -> ToolStatus { .unwrap_or(false); if ok { - tracing::info!(target: "detect", available = true, "ffmpeg detected — video thumbnails and transcoding enabled"); + tracing::info!(target: "rustchan::detect", available = true, "ffmpeg detected — video thumbnails and transcoding enabled"); ToolStatus::Available } else if require_ffmpeg { tracing::error!( - target: "detect", + target: "rustchan::detect", available = false, "ffmpeg required but not installed — set require_ffmpeg = false in settings.toml to disable this check" ); @@ -46,7 +51,7 @@ pub fn detect_ffmpeg(require_ffmpeg: bool) -> ToolStatus { std::process::exit(1); } else { tracing::warn!( - target: "detect", + target: "rustchan::detect", available = false, "ffmpeg not detected — video thumbnails and transcoding disabled" ); @@ -69,13 +74,13 @@ pub fn detect_webp_encoder(ffmpeg_ok: bool) -> bool { if has_webp { tracing::info!( - target: "detect", + target: "rustchan::detect", webp = true, "ffmpeg libwebp encoder available — image to WebP conversion enabled" ); } else { tracing::warn!( - target: "detect", + target: "rustchan::detect", webp = false, "ffmpeg libwebp encoder missing — JPEG/PNG/BMP/TIFF stored in original format" ); @@ -134,13 +139,13 @@ pub fn detect_webm_encoder(ffmpeg_ok: bool) -> bool { if has_webm { tracing::info!( - target: "detect", + target: "rustchan::detect", vp9 = true, opus = true, "ffmpeg VP9 + Opus encoders available — MP4 to WebM transcoding enabled" ); } else { tracing::warn!( - target: "detect", + target: "rustchan::detect", vp9 = has_vp9, opus = has_opus, "ffmpeg VP9/Opus encoders missing — MP4 uploads stored as MP4" @@ -204,47 +209,145 @@ fn webm_install_hint(has_vp9: bool, has_opus: bool) -> String { // No subprocess, no torrc, no hostname file, no polling loop. use arti_client::{config::TorClientConfigBuilder, TorClient}; +use dashmap::DashMap; use futures::StreamExt; +use rand_core::{OsRng, RngCore}; +use std::sync::LazyLock; use tokio::net::TcpStream; use tokio::sync::RwLock; +use tokio_util::sync::CancellationToken; use tor_cell::relaycell::msg::Connected; use tor_hsservice::{config::OnionServiceConfigBuilder, handle_rend_requests, HsId, StreamRequest}; +// ─── Per-stream identity map ────────────────────────────────────────────────── +// +// CRIT-2A fix: maps the ephemeral local port of each Tor proxy connection to a +// random pseudonymous token. +// +// How it works: +// 1. proxy_tor_stream() connects to 127.0.0.1:{bind_port}. The OS assigns +// an ephemeral source port on the connecting end. +// 2. That ephemeral port is what axum's ConnectInfo sees as the peer port +// on the accepted socket. +// 3. A random "tor:" token is inserted into this map keyed by that port. +// 4. ClientIp / extract_ip look up the peer port in this map and return the +// token as the request's "IP address". +// 5. A RAII TokenGuard removes the entry when the proxy task ends. +// +// Result: every Tor stream gets its own isolated bucket for rate limiting, post +// cooldowns, and bans — one Tor user's actions no longer affect all others. +// The token is random per-stream so it cannot track users across reconnections. +pub static TOR_STREAM_TOKENS: LazyLock>> = LazyLock::new(DashMap::new); + +/// Removes a port→token entry from `TOR_STREAM_TOKENS` when dropped. +struct TokenGuard(u16); +impl Drop for TokenGuard { + fn drop(&mut self) { + TOR_STREAM_TOKENS.remove(&self.0); + } +} + /// Spawn the Arti in-process Tor task. /// -/// Returns `Available` immediately. The onion address becomes available in -/// `onion_address` roughly 30 seconds later on first run or ~5 seconds on -/// subsequent runs (consensus served from `arti_cache/`). +/// Returns `Some(JoinHandle)` when Tor support is enabled — the handle should +/// be stored and awaited during graceful shutdown (F-04). The onion address +/// becomes available in `onion_address` roughly 30 seconds after startup on +/// first run, or ~5 seconds on subsequent runs (consensus served from cache). +/// +/// Returns `None` when `enable_tor_support` is false. /// -/// If `enable_tor_support` is false this is a no-op and returns `Missing`. +/// # CRIT-3 fix +/// Accepts a `CancellationToken` so the retry loop and backoff sleep both +/// respond to graceful shutdown. Without this the task had no cancel path and +/// was abandoned with a hard 10-second timeout, leaving Tor circuits open. pub fn detect_tor( enable_tor_support: bool, bind_port: u16, data_dir: &Path, onion_address: Arc>>, -) -> ToolStatus { + cancel: CancellationToken, +) -> Option> { if !enable_tor_support { - return ToolStatus::Missing; + return None; } let data_dir = data_dir.to_path_buf(); - tokio::spawn(async move { - if let Err(e) = run_arti(data_dir, bind_port, onion_address).await { - tracing::error!(target: "detect", error = %e, "Tor: fatal error in Arti task"); + // F-02: Retry loop with cancellation support. + // Backoff: 30 s, 60 s, 120 s, 240 s, 480 s (capped at 2^4 × 30 s). + // HIGH-7 fix: attempt resets to 0 after a healthy long-running session so + // clean exits don't accumulate backoff identically to crash loops. + let handle = tokio::spawn(async move { + // Yield briefly before emitting any output. detect_tor() is called + // after the first-run admin wizard (which is synchronous), but the Tokio + // runtime schedules this task immediately after tokio::spawn returns. + // A short sleep lets the startup banner and any queued tracing events + // flush to the terminal before Tor starts printing, preventing + // interleaving with interactive prompts or the keyboard handler startup. + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + + let mut attempt = 0u32; + loop { + tracing::info!(target: "rustchan::detect", attempt, "Tor: starting Arti"); + let run_start = std::time::Instant::now(); + + // CRIT-3: race run_arti against the shutdown token. + let result = tokio::select! { + r = run_arti(data_dir.clone(), bind_port, onion_address.clone()) => r, + () = cancel.cancelled() => { + tracing::info!(target: "rustchan::detect", "Tor: shutdown signal — exiting"); + *onion_address.write().await = None; + return; + } + }; + + match result { + Ok(()) => { + // HIGH-7: a clean exit after ≥60 s of healthy operation is not + // a crash — reset attempt so backoff stays short on reconnect. + if run_start.elapsed() >= std::time::Duration::from_secs(60) { + attempt = 0; + } + tracing::warn!(target: "rustchan::detect", "Tor: Arti exited cleanly"); + } + Err(e) => { + tracing::error!(target: "rustchan::detect", error = %e, attempt, "Tor: fatal error"); + } + } + + // Clear stale address so UI correctly shows Tor as offline. + *onion_address.write().await = None; + + let backoff = + std::time::Duration::from_secs(30_u64.saturating_mul(1 << attempt.min(4))); + tracing::warn!(target: "rustchan::detect", retry_in = ?backoff, "Tor: scheduling restart"); + + // CRIT-3: also cancel-aware during the backoff sleep. + tokio::select! { + () = tokio::time::sleep(backoff) => {} + () = cancel.cancelled() => { + tracing::info!(target: "rustchan::detect", "Tor: shutdown during backoff — exiting"); + return; + } + } + attempt = attempt.saturating_add(1); } }); tracing::info!( - target: "detect", + target: "rustchan::detect", "Tor: Arti task spawned — bootstrapping in background (first run ~30 s)" ); - ToolStatus::Available + Some(handle) } -/// No-op. Previously killed the tor subprocess. +/// Previously killed the tor subprocess. /// The [`TorClient`] is owned by the `tokio::spawn` task; dropping the runtime -/// closes all circuits cleanly. +/// closes all circuits cleanly. This function is a no-op and should not be called. +#[deprecated( + since = "1.1.0", + note = "Arti lifecycle is managed by the runtime. Dropping TorClient closes circuits cleanly. This fn is a no-op." +)] #[allow(dead_code)] pub const fn kill_tor() {} @@ -269,39 +372,161 @@ async fn run_arti( .build()?; tracing::info!( - target: "detect", + target: "rustchan::detect", cache_dir = %data_dir.join("arti_cache").display(), state_dir = %data_dir.join("arti_state").display(), "Tor: bootstrapping — first run downloads ~2 MB of directory data" ); - let tor_client = TorClient::create_bootstrapped(config) - .await - .map_err(|e| format!("Tor bootstrap failed: {e}"))?; - - tracing::info!(target: "detect", "Tor: connected to the Tor network"); - + // F-01: Wrap bootstrap in a timeout. Without this, a captive portal, + // strict firewall, or Tor directory downtime hangs this task forever. + // HIGH-2 fix: timeout sourced from CONFIG instead of a hardcoded constant + // so operators on censored networks can increase it via settings.toml. + let bootstrap_timeout = + std::time::Duration::from_secs(crate::config::CONFIG.tor_bootstrap_timeout_secs); + // KEEP ALIVE: dropping tor_client closes all Tor circuits and kills the onion service. + let tor_client = + tokio::time::timeout(bootstrap_timeout, TorClient::create_bootstrapped(config)) + .await + .map_err(|_| { + format!( + "Tor bootstrap timed out after {} s — check network connectivity \ + (increase tor_bootstrap_timeout_secs in settings.toml for censored networks)", + crate::config::CONFIG.tor_bootstrap_timeout_secs, + ) + })? + .map_err(|e| format!("Tor bootstrap failed: {e}"))?; + + tracing::info!(target: "rustchan::detect", "Tor: connected to the Tor network"); + + // Security hardening options available in OnionServiceConfigBuilder (Arti 0.40): + // .pow_resistance(...) — proof-of-work DoS resistance + // .rate_limit_num_intro_points — cap introduction point abuse + // Currently left at defaults. Consider exposing these in settings.toml (F-18). + // + // MED-3 fix: nickname sourced from CONFIG.tor_service_nickname (default + // "rustchan") so operators running multiple instances with a shared + // arti_state/ directory can assign distinct names and avoid key collisions. let svc_config = OnionServiceConfigBuilder::default() - .nickname("rustchan".parse()?) + .nickname(crate::config::CONFIG.tor_service_nickname.parse()?) .build()?; + // KEEP ALIVE: dropping onion_service deregisters the service from the Tor network. let (onion_service, rend_requests) = tor_client .launch_onion_service(svc_config)? .ok_or("launch_onion_service returned None — unexpected with code-only config")?; - let hsid = onion_service - .onion_address() - .ok_or("onion_address() returned None immediately after launch")?; + // F-03: onion_address() can return None during early bringup in Arti 0.40; + // key material is not guaranteed to be readable synchronously at launch time. + // Retry up to 10 times at 500 ms intervals (5 s total) before failing. + let hsid = { + let mut found = None; + for i in 0..10u32 { + found = onion_service.onion_address(); + if found.is_some() { + break; + } + tracing::debug!(target: "rustchan::detect", attempt = i, "Tor: waiting for HsId availability"); + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + } + found.ok_or("onion_address() still None after 5 s — service key unavailable")? + }; let onion_name = hsid_to_onion_address(hsid); + publish_onion_address(&onion_name, &data_dir, &onion_address).await; - tracing::info!( - target: "detect", + let mut stream_requests = handle_rend_requests(rend_requests); + + // F-05: Cap concurrent proxy tasks to prevent file-descriptor exhaustion + // under a connection flood. Excess connections are dropped (Arti sends + // RELAY_END automatically when stream_req is dropped). + // HIGH-3 fix: limit sourced from CONFIG so operators can tune it without + // recompiling, e.g. to reduce FD pressure on resource-constrained hosts. + let max_streams = crate::config::CONFIG.tor_max_concurrent_streams; + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(max_streams)); + + // HIGH-5 fix: Arc avoids a String heap allocation per connection. + let local_addr: Arc = Arc::from(format!("127.0.0.1:{bind_port}").as_str()); + + while let Some(stream_req) = stream_requests.next().await { + match std::sync::Arc::clone(&sem).try_acquire_owned() { + Ok(permit) => { + let addr = Arc::clone(&local_addr); + tokio::spawn(async move { + let _permit = permit; // released on drop + // HIGH-4 fix: distinguish infrastructure failures (axum + // unreachable) from normal stream closure (client disconnect, + // EOF). Infrastructure errors go to ERROR; everything else + // stays at DEBUG so the log isn't flooded by normal traffic. + if let Err(e) = proxy_tor_stream(stream_req, &addr).await { + let msg = e.to_string(); + if msg.contains("local TCP connect failed") + || msg.contains("timed out connecting to local HTTP server") + { + tracing::error!( + target: "rustchan::detect", + error = %e, + "Tor: cannot reach local HTTP server — is axum running?" + ); + } else { + tracing::debug!( + target: "rustchan::detect", + error = %e, + "Tor: stream closed" + ); + } + } + }); + } + Err(_) => { + tracing::warn!( + target: "rustchan::detect", + limit = max_streams, + "Tor: stream limit reached — dropping connection" + ); + // Dropping stream_req causes Arti to send a RELAY_END cell. + } + } + } + + // CRIT-5/6: belt-and-suspenders keepalives. Named let-bindings in Rust + // drop at end of their enclosing scope (the function body), not at + // last-use — so tor_client and onion_service are already live here. + // These explicit borrows make the intent unambiguous to future readers + // and guard against any tooling that might warn about "unused" bindings. + let _ = &tor_client; + let _ = &onion_service; + + tracing::warn!( + target: "rustchan::detect", + "Tor: rendezvous stream ended — onion service offline" + ); + Ok(()) +} + +// ─── Onion address publication ──────────────────────────────────────────────── + +/// Log the active onion address, write it to the shared state, and print the +/// TTY banner. Extracted from `run_arti` to keep that function under the +/// clippy line-count limit. +/// +/// MED-7/MED-11: the address is logged at DEBUG only so it never appears in +/// plaintext in JSON log files forwarded to aggregators. Set +/// `RUST_LOG=detect=debug` to see it in logs; the TTY banner and admin panel +/// always show the full address. +async fn publish_onion_address( + onion_name: &str, + data_dir: &std::path::Path, + onion_address: &RwLock>, +) { + tracing::debug!( + target: "rustchan::detect", onion_address = %onion_name, keys_dir = %data_dir.join("arti_state").join("keys").display(), "Tor: hidden service active" ); + tracing::info!(target: "rustchan::detect", "Tor: hidden service active"); - *onion_address.write().await = Some(onion_name.clone()); + *onion_address.write().await = Some(onion_name.to_string()); if crate::logging::is_tty() { crate::logging::console_print_raw(&format!( @@ -319,27 +544,6 @@ async fn run_arti( keys = data_dir.join("arti_state/keys").display(), )); } - - let mut stream_requests = handle_rend_requests(rend_requests); - - while let Some(stream_req) = stream_requests.next().await { - let local_addr = format!("127.0.0.1:{bind_port}"); - tokio::spawn(async move { - if let Err(e) = proxy_tor_stream(stream_req, &local_addr).await { - tracing::debug!( - target: "detect", - error = %e, - "Tor: stream closed" - ); - } - }); - } - - tracing::warn!( - target: "detect", - "Tor: rendezvous stream ended — onion service offline" - ); - Ok(()) } // ─── Connection proxy ───────────────────────────────────────────────────────── @@ -349,7 +553,36 @@ async fn proxy_tor_stream( local_addr: &str, ) -> Result<(), Box> { let mut tor_stream = stream_req.accept(Connected::new_empty()).await?; - let mut local = TcpStream::connect(local_addr).await?; + + // MED-10 fix: increased from 5 s to 15 s — under load the axum TCP accept + // queue can fill and connect() can legitimately take several seconds. + let mut local = tokio::time::timeout( + std::time::Duration::from_secs(15), + TcpStream::connect(local_addr), + ) + .await + .map_err(|_| "timed out connecting to local HTTP server")? + .map_err(|e| format!("local TCP connect failed: {e}"))?; + + // CRIT-2A: Register a per-stream pseudonymous token keyed on the ephemeral + // local port. axum's ConnectInfo sees this port as the peer port on the + // incoming socket, so ClientIp / extract_ip can retrieve the token without + // any HTTP parsing, through keep-alive, across all content types. + let local_port = local.local_addr().map(|a| a.port()).unwrap_or(0); + let token: Arc = { + let mut bytes = [0u8; 16]; + OsRng.fill_bytes(&mut bytes); + Arc::from(format!("tor:{}", hex::encode(bytes)).as_str()) + }; + // _guard removes the map entry when this task ends (connection closed or error). + let _guard = if local_port != 0 { + TOR_STREAM_TOKENS.insert(local_port, Arc::clone(&token)); + Some(TokenGuard(local_port)) + } else { + tracing::debug!(target: "rustchan::detect", "Tor: could not determine local port — stream uses shared bucket"); + None + }; + tokio::io::copy_bidirectional(&mut tor_stream, &mut local).await?; Ok(()) } @@ -372,12 +605,11 @@ fn hsid_to_onion_address(hsid: HsId) -> String { hasher.update(b".onion checksum"); hasher.update(pubkey); hasher.update([version]); - let hash = hasher.finalize(); - - // Destructure via iterator — avoids the indexing_slicing lint. - // Safe: Sha3_256 always produces exactly 32 bytes. - let mut iter = hash.iter().copied(); - let checksum = [iter.next().unwrap_or(0), iter.next().unwrap_or(0)]; + // HIGH-1 fix: use a typed [u8;32] array instead of an iterator with + // unwrap_or(0). Sha3_256 always produces 32 bytes — the fallback was dead + // code that masked potential logic errors if the digest size ever changed. + let hash: [u8; 32] = hasher.finalize().into(); + let checksum = [hash[0], hash[1]]; let mut address_bytes = [0u8; 35]; address_bytes[..32].copy_from_slice(pubkey); @@ -390,3 +622,50 @@ fn hsid_to_onion_address(hsid: HsId) -> String { format!("{encoded}.onion") } + +// ─── Tests ──────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + /// Verify the v3 onion address encoder against a Python-computed reference + /// value for the all-zeros Ed25519 key. + /// + /// Reference: + /// ```python + /// import hashlib, base64 + /// pub = bytes(32); ver = bytes([3]) + /// chk = hashlib.sha3_256(b'.onion checksum' + pub + ver).digest()[:2] + /// raw = pub + chk + ver + /// print(base64.b32encode(raw).decode().lower().rstrip('=') + '.onion') + /// # → aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaam2dqd.onion + /// ``` + #[test] + fn onion_address_format_is_56_chars_plus_dot_onion() { + let zeroed: [u8; 32] = [0u8; 32]; + let hsid = HsId::from(zeroed); + let addr = hsid_to_onion_address(hsid); + + // Verified test vector: Python sha3_256 reference + base32 encoding. + assert_eq!( + addr, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaam2dqd.onion", + "zero-key address must match Python reference implementation" + ); + + assert_eq!(addr.len(), 62, "v3 onion address must be 62 chars total"); + // Use index slicing instead of ends_with / trim_end_matches to avoid + // the clippy::case_sensitive_file_extension_comparisons lint. + // The encoder always produces lowercase output, and the length is + // already asserted above, so fixed-index slicing is safe here. + assert_eq!(&addr[56..], ".onion", "must end with .onion"); + let base32_part = &addr[..56]; + assert_eq!(base32_part.len(), 56, "base32 part must be 56 chars"); + assert!( + base32_part + .chars() + .all(|c| matches!(c, 'a'..='z' | '2'..='7')), + "base32 part must be lowercase a-z2-7 only" + ); + } +} diff --git a/src/logging.rs b/src/logging.rs index fc8583f..b1539c8 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -316,8 +316,24 @@ pub fn init_logging(log_dir: &Path) { let tty = io::stdout().is_terminal(); IS_TTY.store(tty, Ordering::Relaxed); - let env_filter = EnvFilter::try_from_default_env() - .unwrap_or_else(|_| EnvFilter::new("rustchan=info,tower_http=warn")); + let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| { + EnvFilter::new( + // F-08: Explicitly suppress Arti's internal crates in the default + // filter. Without this they emit at DEBUG/TRACE (circuit negotiation, + // guard selection, consensus downloads) — hundreds of lines/minute. + // Operators who need Arti internals can set RUST_LOG=tor_proto=debug. + "rustchan=info,\ + tower_http=warn,\ + arti_client=warn,\ + tor_proto=warn,\ + tor_circmgr=warn,\ + tor_dirmgr=warn,\ + tor_guardmgr=warn,\ + tor_chanmgr=warn,\ + tor_hsservice=warn,\ + tor_keymgr=warn", + ) + }); let terminal_layer = tracing_subscriber::fmt::layer() .event_format(TerminalFormatter) diff --git a/src/middleware/mod.rs b/src/middleware/mod.rs index a7c336b..d3bbd41 100644 --- a/src/middleware/mod.rs +++ b/src/middleware/mod.rs @@ -416,10 +416,27 @@ pub fn extract_ip(req: &Request) -> String { } } - // Direct connection IP (not behind proxy, or proxy headers absent) - req.extensions() - .get::>() - .map_or_else(|| "unknown".to_string(), |ci| ci.0.ip().to_string()) + // Direct connection IP (not behind proxy, or proxy headers absent). + let ci = req + .extensions() + .get::>(); + let peer = ci.map(|c| c.0); + + // CRIT-2A: when the TCP peer is loopback and Tor support is active, look + // up the peer port in TOR_STREAM_TOKENS. Each Tor stream registers a unique + // pseudonymous token so that Tor users get isolated rate-limit and ban + // buckets instead of all sharing "127.0.0.1". + if CONFIG.enable_tor_support { + if let Some(addr) = peer { + if addr.ip().is_loopback() { + if let Some(token) = crate::detect::TOR_STREAM_TOKENS.get(&addr.port()) { + return token.value().to_string(); + } + } + } + } + + peer.map_or_else(|| "unknown".to_string(), |a| a.ip().to_string()) } /// Validate CSRF token from a form against the cookie. @@ -546,10 +563,25 @@ where } } // Direct connection (or proxy headers absent). - let ip = parts + let ci = parts .extensions - .get::>() - .map_or_else(|| "unknown".to_string(), |ci| ci.0.ip().to_string()); + .get::>(); + let peer = ci.map(|c| c.0); + + // CRIT-2A: same token lookup as extract_ip — keeps both code paths + // consistent. When the TCP peer is loopback and Tor is enabled, return + // the per-stream pseudonymous token instead of "127.0.0.1". + if CONFIG.enable_tor_support { + if let Some(addr) = peer { + if addr.ip().is_loopback() { + if let Some(token) = crate::detect::TOR_STREAM_TOKENS.get(&addr.port()) { + return Ok(Self(token.value().to_string())); + } + } + } + } + + let ip = peer.map_or_else(|| "unknown".to_string(), |a| a.ip().to_string()); Ok(Self(ip)) } } diff --git a/src/server/server.rs b/src/server/server.rs index 0fa5e46..36ee14b 100644 --- a/src/server/server.rs +++ b/src/server/server.rs @@ -218,10 +218,20 @@ pub async fn run_server(port_override: Option, chan_net: bool) -> anyhow::R // Derive bind_port from `bind_addr` (which already incorporates port_override). // rsplit_once(':') handles both IPv4 ("0.0.0.0:9000") and IPv6 ("[::1]:9000"). + // F-07: Log a warning if parsing fails so the operator knows Tor proxy is + // using a fallback port that may not match the actual HTTP listener. let bind_port = bind_addr .rsplit_once(':') .and_then(|(_, p)| p.parse::().ok()) - .unwrap_or(8080); + .unwrap_or_else(|| { + tracing::warn!( + target: "server", + bind_addr = %bind_addr, + fallback = 8080, + "Could not parse port from bind_addr — Tor proxy will use port 8080" + ); + 8080 + }); // CRIT-1 FIX: Capture JoinHandles from start_worker_pool so the shutdown // sequence can await each worker instead of blindly sleeping for 10 s. @@ -247,16 +257,9 @@ pub async fn run_server(port_override: Option, chan_net: bool) -> anyhow::R onion_address: std::sync::Arc::new(tokio::sync::RwLock::new(None)), }; - // Tor: spawn Arti in-process. The onion address is written to - // state.onion_address once Arti has bootstrapped and the hidden service - // is active (~30 s first run, ~5 s on subsequent runs). - crate::detect::detect_tor( - CONFIG.enable_tor_support, - bind_port, - &data_dir, - state.onion_address.clone(), - ); - // Keep a reference to the job queue cancel token for graceful shutdown (#7). + // worker_cancel is the shutdown token threaded through all background tasks + // and the Tor task. Declared here so it is available to background tasks + // spawned below. detect_tor is called later, after the first-run wizard. let worker_cancel = state.job_queue.cancel.clone(); let start_time = Instant::now(); @@ -522,6 +525,20 @@ pub async fn run_server(port_override: Option, chan_net: bool) -> anyhow::R } } + // Tor: spawn Arti in-process AFTER the first-run wizard so that Arti's + // bootstrapping log events do not interleave with wizard prompts. + // console_prompt() releases CONSOLE_MUTEX before blocking on read_line, + // so any concurrent tracing::info! from the Tor task would print mid-prompt. + // Arti bootstrapping takes ~30 s regardless, so starting it here costs nothing. + // F-04: Store the handle so it can be awaited during graceful shutdown. + let tor_handle = crate::detect::detect_tor( + CONFIG.enable_tor_support, + bind_port, + &data_dir, + state.onion_address.clone(), + worker_cancel.clone(), + ); + super::console::spawn_keyboard_handler(pool, start_time); if chan_net { @@ -560,6 +577,16 @@ pub async fn run_server(port_override: Option, chan_net: bool) -> anyhow::R let _ = tokio::time::timeout(shutdown_timeout, handle).await; } + // CRIT-3 FIX: worker_cancel.cancel() above already signals the Tor task's + // CancellationToken, so it will exit its select! loop promptly instead of + // sleeping through a multi-minute backoff. The 15-second safety-net timeout + // below is only a last resort for the in-flight copy_bidirectional on any + // active stream — Arti sends RELAY_END cells synchronously on drop, which + // completes well within this window under normal conditions. + if let Some(h) = tor_handle { + let _ = tokio::time::timeout(Duration::from_secs(15), h).await; + } + tracing::info!(target: "server", "Server shut down gracefully."); Ok(()) } @@ -870,9 +897,65 @@ fn build_router(state: AppState) -> Router { }, ), ) + // HIGH-9: Inject `Onion-Location` response header when the onion service + // is active. Tor Browser reads this header on clearnet responses and + // prompts the user to switch to the .onion address automatically. + // Only injected when enable_tor_support=true and the address is known. + .layer(axum_middleware::from_fn_with_state( + state.clone(), + onion_location_middleware, + )) .with_state(state) } +/// Inject the `Onion-Location` response header when the Tor hidden service is +/// active and the request arrived over clearnet (not already via .onion). +/// +/// Tor Browser reads this header and prompts the user to switch to the .onion +/// address automatically, improving privacy without requiring the user to know +/// the onion address in advance. +/// +/// The header is suppressed when: +/// - `enable_tor_support` is false (no onion service running) +/// - The onion address is not yet known (Arti still bootstrapping) +/// - The request already came in via the onion address (no double-redirect) +/// +/// Spec: +async fn onion_location_middleware( + axum::extract::State(state): axum::extract::State, + req: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + // Skip if Tor is not enabled. + if !crate::config::CONFIG.enable_tor_support { + return next.run(req).await; + } + + // Read the onion address under a short-lived read lock before await. + let maybe_addr = state.onion_address.read().await.clone(); + + let mut resp = next.run(req).await; + + if let Some(addr) = maybe_addr { + // Only inject on HTML responses — static assets, JSON, and media do + // not benefit from the header and it adds noise to every response. + let is_html = resp + .headers() + .get(header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .is_some_and(|ct| ct.contains("text/html")); + + if is_html { + if let Ok(val) = header::HeaderValue::from_str(&format!("http://{addr}")) { + resp.headers_mut() + .insert(header::HeaderName::from_static("onion-location"), val); + } + } + } + + resp +} + /// Middleware that adds `Strict-Transport-Security` only when the connection /// is confirmed to be HTTPS (RFC 6797 §7.2). Checks both the URI scheme /// (set by some reverse proxies) and the `X-Forwarded-Proto` header.