From 05f0efa946f71bfa010e877e101ed8715ebb36ef Mon Sep 17 00:00:00 2001 From: Andreas Zwinkau Date: Tue, 24 Mar 2026 14:42:46 +0100 Subject: [PATCH 1/5] Fix any_folder missing files issue --- docs.bzl | 19 ++++++++++--------- docs/how-to/any_folder.rst | 19 +++++++++++++++++++ src/extensions/score_any_folder/__init__.py | 11 ++++++++++- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/docs.bzl b/docs.bzl index cd41a4c8..439a26d6 100644 --- a/docs.bzl +++ b/docs.bzl @@ -120,7 +120,7 @@ def _missing_requirements(deps): fail(msg) fail("This case should be unreachable?!") -def docs(source_dir = "docs", data = [], deps = [], scan_code = []): +def docs(source_dir = "docs", data = [], deps = [], scan_code = [], source_dir_extras = []): """Creates all targets related to documentation. By using this function, you'll get any and all updates for documentation targets in one place. @@ -130,6 +130,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []): data: Additional data files to include in the documentation build. deps: Additional dependencies for the documentation build. scan_code: List of code targets to scan for source code links. + source_dir_extras: Additional source targets outside of source_dir (any_folder) """ call_path = native.package_name() @@ -167,7 +168,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []): source_dir + "/**/*.csv", source_dir + "/**/*.inc", "more_docs/**/*.rst", - ], allow_empty = True), + ], allow_empty = True) + source_dir_extras, visibility = ["//visibility:public"], ) @@ -181,7 +182,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []): name = "docs", tags = ["cli_help=Build documentation:\nbazel run //:docs"], srcs = ["@score_docs_as_code//src:incremental.py"], - data = data + [":sourcelinks_json"], + data = data + source_dir_extras + [":sourcelinks_json"], deps = deps, env = { "SOURCE_DIRECTORY": source_dir, @@ -195,7 +196,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []): name = "docs_combo", tags = ["cli_help=Build full documentation with all dependencies:\nbazel run //:docs_combo"], srcs = ["@score_docs_as_code//src:incremental.py"], - data = data_with_docs_sources + [":merged_sourcelinks"], + data = data_with_docs_sources + source_dir_extras + [":merged_sourcelinks"], deps = deps, env = { "SOURCE_DIRECTORY": source_dir, @@ -215,7 +216,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []): name = "docs_link_check", tags = ["cli_help=Verify Links inside Documentation:\nbazel run //:link_check\n (Note: this could take a long time)"], srcs = ["@score_docs_as_code//src:incremental.py"], - data = data, + data = data + source_dir_extras, deps = deps, env = { "SOURCE_DIRECTORY": source_dir, @@ -228,7 +229,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []): name = "docs_check", tags = ["cli_help=Verify documentation:\nbazel run //:docs_check"], srcs = ["@score_docs_as_code//src:incremental.py"], - data = data + [":sourcelinks_json"], + data = data + source_dir_extras + [":sourcelinks_json"], deps = deps, env = { "SOURCE_DIRECTORY": source_dir, @@ -242,7 +243,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []): name = "live_preview", tags = ["cli_help=Live preview documentation in the browser:\nbazel run //:live_preview"], srcs = ["@score_docs_as_code//src:incremental.py"], - data = data + [":sourcelinks_json"], + data = data + source_dir_extras + [":sourcelinks_json"], deps = deps, env = { "SOURCE_DIRECTORY": source_dir, @@ -256,7 +257,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []): name = "live_preview_combo_experimental", tags = ["cli_help=Live preview full documentation with all dependencies in the browser:\nbazel run //:live_preview_combo_experimental"], srcs = ["@score_docs_as_code//src:incremental.py"], - data = data_with_docs_sources + [":merged_sourcelinks"], + data = data_with_docs_sources + source_dir_extras + [":merged_sourcelinks"], deps = deps, env = { "SOURCE_DIRECTORY": source_dir, @@ -272,7 +273,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []): venv_name = ".venv_docs", reqs = deps, # Add dependencies to ide_support, so esbonio has access to them. - data = data, + data = data + source_dir_extras, ) sphinx_docs( diff --git a/docs/how-to/any_folder.rst b/docs/how-to/any_folder.rst index 0e3c1427..da095deb 100644 --- a/docs/how-to/any_folder.rst +++ b/docs/how-to/any_folder.rst @@ -18,6 +18,7 @@ If you have ``docs/component/overview.rst``, for example, you can include the component documentation via ``toctree``: .. code-block:: rst + :caption: some rst file .. toctree:: @@ -27,3 +28,21 @@ Only relative links are allowed. The symlinks will show up in your sources. **Don't commit the symlinks to git!** + +Bazel +----- + +When building with Bazel, declare the mapped directories as ``source_dir_extras`` +in your ``docs()`` call so Bazel tracks them as dependencies: + +.. code-block:: python + :caption: BUILD + + docs( + source_dir = "docs", + source_dir_extras = ["//score/containers:docs_sources"], + ... + ) + +This is necessary for sandboxed builds. +For example, when other modules use your documentation's ``needs.json`` as a dependency. diff --git a/src/extensions/score_any_folder/__init__.py b/src/extensions/score_any_folder/__init__.py index 13526b77..9e28a784 100644 --- a/src/extensions/score_any_folder/__init__.py +++ b/src/extensions/score_any_folder/__init__.py @@ -100,7 +100,16 @@ def _create_symlinks(app: Sphinx) -> None: continue link.parent.mkdir(parents=True, exist_ok=True) - link.symlink_to(source) + try: + link.symlink_to(source) + except OSError as exc: + logger.error( + "score_any_folder: failed to create symlink %s -> %s: %s", + link, + source, + exc, + ) + continue created_links.add(link) logger.debug("score_any_folder: created symlink %s -> %s", link, source) From 8ba08cc71199e2f7a5c74076e1e391d7832712ff Mon Sep 17 00:00:00 2001 From: Andreas Zwinkau Date: Tue, 24 Mar 2026 14:59:58 +0100 Subject: [PATCH 2/5] Apply fix to docs_as_code itself --- BUILD | 3 +++ src/extensions/docs/BUILD | 5 +++++ 2 files changed, 8 insertions(+) create mode 100644 src/extensions/docs/BUILD diff --git a/BUILD b/BUILD index c84b7796..34693876 100644 --- a/BUILD +++ b/BUILD @@ -38,6 +38,9 @@ docs( "//src:all_sources", ], source_dir = "docs", + source_dir_extras = [ + "//src/extensions/docs:doc_sources_extra", + ], ) cli_helper( diff --git a/src/extensions/docs/BUILD b/src/extensions/docs/BUILD new file mode 100644 index 00000000..b18eb770 --- /dev/null +++ b/src/extensions/docs/BUILD @@ -0,0 +1,5 @@ +filegroup( + name = "doc_sources_extra", + srcs = ["data_flow.png"] + glob(["*.rst", "*.md"]), + visibility = ["//visibility:public"], +) From 5576bbed496868d8401b9b91f36b011ff443ff2a Mon Sep 17 00:00:00 2001 From: Andreas Zwinkau Date: Wed, 25 Mar 2026 17:29:43 +0100 Subject: [PATCH 3/5] tinkering but doesn't work yet --- docs/how-to/any_folder.rst | 3 +- src/extensions/docs/any_folder.rst | 17 ++- src/extensions/score_any_folder/__init__.py | 137 +++++++++++++----- .../tests/test_score_any_folder.py | 127 ++++++++++++++++ 4 files changed, 247 insertions(+), 37 deletions(-) diff --git a/docs/how-to/any_folder.rst b/docs/how-to/any_folder.rst index da095deb..7b2b2b37 100644 --- a/docs/how-to/any_folder.rst +++ b/docs/how-to/any_folder.rst @@ -12,7 +12,8 @@ You can symlink the folders by adding to your ``conf.py``: "../score/containers/docs": "component/containers", } -With this configuration, all files in ``score/containers/docs/`` become available at ``docs/component/containers/``. +All files in ``score/containers/docs/`` become available at ``docs/component/containers/``. +Include them via ``toctree`` as usual. If you have ``docs/component/overview.rst``, for example, you can include the component documentation via ``toctree``: diff --git a/src/extensions/docs/any_folder.rst b/src/extensions/docs/any_folder.rst index d69864ba..ab65ba41 100644 --- a/src/extensions/docs/any_folder.rst +++ b/src/extensions/docs/any_folder.rst @@ -24,8 +24,23 @@ Sphinx then discovers and buildsthose files as if they were part of ``docs/`` fr The extension hooks into the ``builder-inited`` event, which fires before Sphinx reads any documents. +Configuration reference +----------------------- + +``score_any_folder_mapping`` + *dict[str, str]*, default ``{}`` + + Maps source directories to symlink paths, both relative to ``confdir``. + Applied on every Sphinx build. + + .. code-block:: python + + score_any_folder_mapping = { + "../src/my_module/docs": "my_module", + } + Difference to Sphinx-Collections --------------------------------- +--------------------------------- The extension `sphinx-collections `_ is very similar to this extension. diff --git a/src/extensions/score_any_folder/__init__.py b/src/extensions/score_any_folder/__init__.py index 9e28a784..c28a880d 100644 --- a/src/extensions/score_any_folder/__init__.py +++ b/src/extensions/score_any_folder/__init__.py @@ -28,14 +28,28 @@ The extension creates the symlinks on ``builder-inited``, before Sphinx starts reading any documents. -Existing correct symlinks are left in place(idempotent); +Existing correct symlinks are left in place (idempotent); a symlink pointing to the wrong target is replaced. Symlinks created by this extension are removed again on ``build-finished``. Misconfigured pairs (absolute paths, non-symlink path at the target location) are logged as errors and skipped. + +Combo builds +------------ + +When a combo build mounts external modules via ``sphinx_collections``, +those modules may have their own ``score_any_folder_mapping`` in their +``conf.py``. This extension automatically discovers those files by scanning +``confdir`` subdirectories after the primary symlink pass and applies their +mappings with paths resolved relative to each module's directory. + +No extra configuration is required. The handler is registered at event +priority 600 (above the default 500) to ensure it runs after +``sphinx_collections`` has mounted its collections. """ +import ast from pathlib import Path from sphinx.application import Sphinx @@ -45,9 +59,11 @@ _APP_ATTRIBUTE = "_score_any_folder_created_links" + def setup(app: Sphinx) -> dict[str, str | bool]: app.add_config_value("score_any_folder_mapping", default={}, rebuild="env") - app.connect("builder-inited", _create_symlinks) + # Priority 600 > default 500: run after sphinx_collections has mounted modules. + app.connect("builder-inited", _create_symlinks, priority=600) app.connect("build-finished", _cleanup_symlinks) return { "version": "0.1", @@ -56,11 +72,39 @@ def setup(app: Sphinx) -> dict[str, str | bool]: } -def _symlink_pairs(app: Sphinx) -> list[tuple[Path, Path]]: - """Return ``(resolved_source, link_path)`` pairs from the mapping.""" - confdir = Path(app.confdir) +def _extract_mapping_from_conf(conf_path: Path) -> dict[str, str]: + """Safely extract ``score_any_folder_mapping`` from a ``conf.py`` file. + + Uses ``ast.literal_eval`` so no arbitrary code is executed. + Returns an empty dict if the key is absent or cannot be parsed. + """ + try: + tree = ast.parse(conf_path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if not isinstance(node, ast.Assign): + continue + for target in node.targets: + if ( + isinstance(target, ast.Name) + and target.id == "score_any_folder_mapping" + ): + return ast.literal_eval(node.value) + except Exception as exc: # noqa: BLE001 + logger.debug( + "score_any_folder: could not extract mapping from %s: %s", + conf_path, + exc, + ) + return {} + + +def _symlink_pairs(confdir: Path, mapping: dict[str, str]) -> list[tuple[Path, Path]]: + """Return ``(resolved_source, link_path)`` pairs from *mapping*. + + Entries with absolute paths are logged as errors and skipped. + """ pairs = [] - for source_rel, target_rel in app.config.score_any_folder_mapping.items(): + for source_rel, target_rel in mapping.items(): if Path(source_rel).is_absolute(): logger.error( "score_any_folder: source path must be relative, got: %r; skipping", @@ -79,39 +123,62 @@ def _symlink_pairs(app: Sphinx) -> list[tuple[Path, Path]]: return pairs +def _maybe_create_symlink(source: Path, link: Path, created_links: set[Path]) -> None: + """Create a symlink at *link* pointing to *source*, if needed. + + Handles the idempotent / stale-symlink / existing-path cases and logs + errors without raising. Successfully created links are added to + *created_links* for later cleanup. + """ + if link.is_symlink(): + if link.resolve() == source: + logger.debug("score_any_folder: symlink already correct: %s", link) + return + logger.info("score_any_folder: replacing stale symlink %s -> %s", link, source) + link.unlink() + elif link.exists(): + logger.error( + "score_any_folder: target path already exists and is not a symlink: " + "%s; skipping", + link, + ) + return + + link.parent.mkdir(parents=True, exist_ok=True) + try: + link.symlink_to(source) + except OSError as exc: + logger.error( + "score_any_folder: failed to create symlink %s -> %s: %s", + link, + source, + exc, + ) + return + created_links.add(link) + logger.debug("score_any_folder: created symlink %s -> %s", link, source) + + def _create_symlinks(app: Sphinx) -> None: created_links: set[Path] = set() + confdir = Path(app.confdir) - for source, link in _symlink_pairs(app): - if link.is_symlink(): - if link.resolve() == source: - logger.debug("score_any_folder: symlink already correct: %s", link) - continue - logger.info( - "score_any_folder: replacing stale symlink %s -> %s", link, source - ) - link.unlink() - elif link.exists(): - logger.error( - "score_any_folder: target path already exists and is not a symlink: " - "%s; skipping", - link, - ) - continue - - link.parent.mkdir(parents=True, exist_ok=True) - try: - link.symlink_to(source) - except OSError as exc: - logger.error( - "score_any_folder: failed to create symlink %s -> %s: %s", - link, - source, - exc, - ) + # Primary pass — mappings defined in the main conf.py. + for source, link in _symlink_pairs(confdir, app.config.score_any_folder_mapping): + _maybe_create_symlink(source, link, created_links) + + # Secondary pass — auto-discover conf.py files in subdirectories. + # Picks up modules mounted by sphinx_collections (or any other mechanism). + # Running at priority 600 ensures sphinx_collections has already mounted + # its collections before we scan. + for conf_py in sorted(confdir.rglob("conf.py")): + if conf_py.parent == confdir: + continue # skip the main conf.py + module_mapping = _extract_mapping_from_conf(conf_py) + if not module_mapping: continue - created_links.add(link) - logger.debug("score_any_folder: created symlink %s -> %s", link, source) + for source, link in _symlink_pairs(conf_py.parent, module_mapping): + _maybe_create_symlink(source, link, created_links) setattr(app, _APP_ATTRIBUTE, created_links) diff --git a/src/extensions/score_any_folder/tests/test_score_any_folder.py b/src/extensions/score_any_folder/tests/test_score_any_folder.py index 0f04d6ce..3362ba1b 100644 --- a/src/extensions/score_any_folder/tests/test_score_any_folder.py +++ b/src/extensions/score_any_folder/tests/test_score_any_folder.py @@ -16,6 +16,7 @@ from pathlib import Path import pytest +from score_any_folder import _extract_mapping_from_conf from sphinx.testing.util import SphinxTestApp @@ -68,6 +69,49 @@ def _factory(mapping: dict[str, str]) -> SphinxTestApp: app.cleanup() +# --------------------------------------------------------------------------- +# _extract_mapping_from_conf +# --------------------------------------------------------------------------- + + +def test_extract_mapping_returns_dict(tmp_path: Path) -> None: + conf = tmp_path / "conf.py" + conf.write_text('score_any_folder_mapping = {"../src": "src"}\n') + assert _extract_mapping_from_conf(conf) == {"../src": "src"} + + +def test_extract_mapping_missing_key_returns_empty(tmp_path: Path) -> None: + conf = tmp_path / "conf.py" + conf.write_text("project = 'test'\n") + assert _extract_mapping_from_conf(conf) == {} + + +def test_extract_mapping_non_literal_value_returns_empty(tmp_path: Path) -> None: + conf = tmp_path / "conf.py" + conf.write_text("score_any_folder_mapping = dict(src='src')\n") + assert _extract_mapping_from_conf(conf) == {} + + +def test_extract_mapping_syntax_error_returns_empty(tmp_path: Path) -> None: + conf = tmp_path / "conf.py" + conf.write_text("score_any_folder_mapping = {this is not valid python\n") + assert _extract_mapping_from_conf(conf) == {} + + +def test_extract_mapping_multiple_assignments_returns_first(tmp_path: Path) -> None: + conf = tmp_path / "conf.py" + conf.write_text( + 'score_any_folder_mapping = {"../a": "a"}\n' + 'score_any_folder_mapping = {"../b": "b"}\n' + ) + assert _extract_mapping_from_conf(conf) == {"../a": "a"} + + +# --------------------------------------------------------------------------- +# Primary symlink behaviour +# --------------------------------------------------------------------------- + + def test_symlink_exposes_files_at_target_path( make_sphinx_app: Callable[[dict[str, str]], SphinxTestApp], docs_dir: Path, @@ -175,3 +219,86 @@ def test_target_in_subfolder( link = docs_dir / "foo" / "other" assert link.is_symlink() assert link.resolve() == src_docs.resolve() + + +# --------------------------------------------------------------------------- +# Auto-discovery of module conf.py files (combo build support) +# --------------------------------------------------------------------------- + + +def test_autodiscovery_applies_module_mapping( + make_sphinx_app: Callable[[dict[str, str]], SphinxTestApp], + docs_dir: Path, + tmp_path: Path, +) -> None: + """A conf.py found in a subdirectory has its mapping applied automatically.""" + # Simulate a sphinx_collections mount at docs/_collections/module/ + module_docs = docs_dir / "_collections" / "module" + module_docs.mkdir(parents=True) + containers = tmp_path / "module_repo" / "containers" / "docs" + containers.mkdir(parents=True) + (containers / "page.rst").write_text("Container Page\n==============\n") + (module_docs / "conf.py").write_text( + 'score_any_folder_mapping = {"../../../module_repo/containers/docs":' + ' "component/containers"}\n' + ) + + make_sphinx_app({}) + + link = module_docs / "component" / "containers" + assert link.is_symlink() + assert link.resolve() == containers.resolve() + assert (link / "page.rst").read_text() == "Container Page\n==============\n" + + +def test_autodiscovery_cleans_up_secondary_symlinks( + make_sphinx_app: Callable[[dict[str, str]], SphinxTestApp], + docs_dir: Path, + tmp_path: Path, +) -> None: + """Secondary symlinks from auto-discovered modules are removed on build-finished.""" + module_docs = docs_dir / "_collections" / "module" + module_docs.mkdir(parents=True) + external = tmp_path / "external" + external.mkdir() + (module_docs / "conf.py").write_text( + 'score_any_folder_mapping = {"../../../external": "ext"}\n' + ) + + make_sphinx_app({}).build() + + assert not (module_docs / "ext").exists() + + +def test_autodiscovery_ignores_conf_without_mapping( + make_sphinx_app: Callable[[dict[str, str]], SphinxTestApp], + docs_dir: Path, +) -> None: + """A subdirectory conf.py with no score_any_folder_mapping produces no symlinks.""" + module_docs = docs_dir / "_collections" / "module" + module_docs.mkdir(parents=True) + (module_docs / "conf.py").write_text("project = 'test'\n") + + make_sphinx_app({}).build() + + assert [p for p in module_docs.iterdir() if p.is_symlink()] == [] + + +def test_autodiscovery_nested_conf( + make_sphinx_app: Callable[[dict[str, str]], SphinxTestApp], + docs_dir: Path, + tmp_path: Path, +) -> None: + """Auto-discovery works for conf.py files nested more than one level deep.""" + nested_docs = docs_dir / "_collections" / "org" / "module" / "docs" + nested_docs.mkdir(parents=True) + external = tmp_path / "external" + external.mkdir() + (nested_docs / "conf.py").write_text( + 'score_any_folder_mapping = {"../../../../../external": "ext"}\n' + ) + + make_sphinx_app({}) + + assert (nested_docs / "ext").is_symlink() + assert (nested_docs / "ext").resolve() == external.resolve() From ae9260877a29bf2f4b0fe3f2d1e460300d439b45 Mon Sep 17 00:00:00 2001 From: Andreas Zwinkau Date: Thu, 26 Mar 2026 15:17:48 +0100 Subject: [PATCH 4/5] Now it works --- docs.bzl | 1 + src/extensions/score_any_folder/__init__.py | 26 ++++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/docs.bzl b/docs.bzl index 439a26d6..ad6f1426 100644 --- a/docs.bzl +++ b/docs.bzl @@ -167,6 +167,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], source_dir_e source_dir + "/**/*.json", source_dir + "/**/*.csv", source_dir + "/**/*.inc", + source_dir + "/conf.py", "more_docs/**/*.rst", ], allow_empty = True) + source_dir_extras, visibility = ["//visibility:public"], diff --git a/src/extensions/score_any_folder/__init__.py b/src/extensions/score_any_folder/__init__.py index c28a880d..11529879 100644 --- a/src/extensions/score_any_folder/__init__.py +++ b/src/extensions/score_any_folder/__init__.py @@ -50,6 +50,7 @@ """ import ast +import os from pathlib import Path from sphinx.application import Sphinx @@ -171,12 +172,35 @@ def _create_symlinks(app: Sphinx) -> None: # Picks up modules mounted by sphinx_collections (or any other mechanism). # Running at priority 600 ensures sphinx_collections has already mounted # its collections before we scan. - for conf_py in sorted(confdir.rglob("conf.py")): + # NOTE: Path.rglob() in Python ≤ 3.12 does NOT follow symlinked directories, + # so we use os.walk(followlinks=True) instead. + conf_py_paths = sorted( + Path(dirpath) / "conf.py" + for dirpath, _dirs, filenames in os.walk(confdir, followlinks=True) + if "conf.py" in filenames + ) + for conf_py in conf_py_paths: if conf_py.parent == confdir: continue # skip the main conf.py module_mapping = _extract_mapping_from_conf(conf_py) if not module_mapping: continue + # Exclude source directories from direct Sphinx indexing. + # sphinx_collections mounts the whole module directory, so source_dir_extras + # files appear at two paths: directly through the mount AND via the symlink. + # Adding the direct path to exclude_patterns prevents duplicate-label warnings. + for source_rel in module_mapping: + if not Path(source_rel).is_absolute(): + unresolved = Path(os.path.normpath(conf_py.parent / source_rel)) + if unresolved.is_relative_to(confdir): + rel = str(unresolved.relative_to(confdir)) + if rel not in app.config.exclude_patterns: + app.config.exclude_patterns.append(rel) + logger.debug( + "score_any_folder: excluding direct path %s" + " (accessible via symlink instead)", + rel, + ) for source, link in _symlink_pairs(conf_py.parent, module_mapping): _maybe_create_symlink(source, link, created_links) From 7eb9df7b86cdbcb2c23ccba333e5751fce8ed4d7 Mon Sep 17 00:00:00 2001 From: Andreas Zwinkau Date: Thu, 26 Mar 2026 15:57:40 +0100 Subject: [PATCH 5/5] Add concept documentation about any_folder --- docs/concepts/any_folder.rst | 125 +++++++++++++++++++++++++++++++++++ docs/concepts/index.rst | 1 + 2 files changed, 126 insertions(+) create mode 100644 docs/concepts/any_folder.rst diff --git a/docs/concepts/any_folder.rst b/docs/concepts/any_folder.rst new file mode 100644 index 00000000..348f30ab --- /dev/null +++ b/docs/concepts/any_folder.rst @@ -0,0 +1,125 @@ +.. _any_folder_concept: + +Any Folder +========== + +The goal is simple: developers should be able to place documentation files +anywhere in the repository, not only in ``docs/``. +A component developer writing ``src/my_component/`` naturally wants the component's docs +to live next to the code in ``src/my_component/docs/``. + +Sphinx, however, only reads files inside its source directory (``confdir``,usually ``docs/``). +The ``score_any_folder`` extension bridges this gap +by creating temporary symlinks inside ``confdir`` that point to the external directories. +Sphinx then discovers the files as if they were always there. + +The fundamental conflict: two build modes +----------------------------------------- + +The difficulty is that docs-as-code supports two structurally different build modes +that pull in opposite directions. + +**Live/incremental builds** +(``bazel run :docs``, ``bazel run :live_preview``, Esbonio) +run Sphinx directly on the developer's real workspace filesystem. +Every file in the repository is accessible by its real path. +Symlinks created by ``score_any_folder`` work naturally here. + +**Combo builds** (``bazel run :docs_combo``) +aggregate documentation from multiple external repositories into a single Sphinx run. +External modules are not present as real directories on the filesystem; +they are mounted from Bazel runfiles by ``sphinx_collections``. +Each module's file tree appears under +``_collections/{module}/`` as a directory symlink into the runfiles. + +.. mermaid:: + + graph LR + subgraph a["Module A"] + LIVE[":live_preview"] + SRCS["docs/**"] + EXT["src/extensions/docs/**"] + CONFA["conf.py"] + LIVE --> CONFA + CONFA -.-> EXT + end + + subgraph b["Module B"] + COMBO[":docs_combo"] + SRCS2["docs/**"] + CONFB["conf.py"] + COMBO --> CONFB + end + + LIVE --> SRCS + COMBO --> SRCS + COMBO --> SRCS2 + +The conflict: ``score_any_folder`` is designed to create symlinks based on +the ``score_any_folder_mapping`` in the *active* ``conf.py``. +In a combo build the active conf.py belongs to the aggregating project, +not to any external module. +Each mounted module has its own ``conf.py`` with its own mapping, +but nothing would apply it — +so the symlinks for those modules are never created, and their docs are broken. + +How the conflict is resolved +----------------------------- + +The resolution has three cooperating parts. + +#. **Runtime symlinks (score_any_folder).** + ``score_any_folder`` runs at Sphinx event priority 600, + after ``sphinx_collections`` (priority 500) has already created all mounts. + It then scans ``confdir`` for ``conf.py`` files in *subdirectories*, + which includes the mounted modules' conf files. + For each one it finds, it extracts the ``score_any_folder_mapping`` + and applies the symlink mapping relative to that module's directory — + exactly as if Sphinx were building that module standalone. + +#. **Deduplication (exclude_patterns).** + ``sphinx_collections`` mounts a module's *entire* runfiles tree, not just its ``docs/`` directory. + This means files from ``source_dir_extras`` (e.g. ``src/extensions/docs/``) are visible to Sphinx at two paths: + directly through the mount *and* through the symlink just created. + To prevent duplicate-label errors, + ``score_any_folder`` adds the direct paths to Sphinx's ``exclude_patterns`` + so only the symlinked path is indexed. + +#. **Bazel dependency declarations (source_dir_extras).** + For hermetic ``needs_json`` builds, Bazel must know about every input file before the build starts. + Files reachable only through runtime symlinks are invisible to Bazel's dependency analysis. + The ``source_dir_extras`` parameter of ``docs()`` lets a module declare these external directories + as explicit Bazel filegroup targets. + For combo builds, ``docs_sources`` also includes ``conf.py`` itself + so that the auto-discovery scan in step 1 can find and read it from within the mounted runfiles. + +Architectural risks +-------------------- + +This design works today but rests on several assumptions that could break silently. + +**Symlinks into runfiles.** +Secondary symlinks are created inside the Bazel runfiles tree +(because the sphinx_collections mount is itself a symlink into it). +This works because ``bazel run`` currently provides a writable runfiles tree. +If Bazel ever makes the runfiles read-only, symlink creation will fail. + +**exclude_patterns is a workaround for a mount granularity problem.** +The real cause of the duplication is that ``sphinx_collections`` mounts the full module root, not just ``docs/``. +The ``exclude_patterns`` approach is a compensating hack. +If Sphinx changes how it processes ``exclude_patterns``, +or if ``sphinx_collections`` changes how it creates mounts, +the deduplication can silently stop working. +The symptom would be a flood of duplicate-label warnings in combo builds. + +**Timing dependency.** +The entire secondary-scan mechanism depends on ``sphinx_collections`` +completing its mounts before ``score_any_folder`` scans for conf.py files. +This is enforced by Sphinx event priority (600 vs. 500), an internal detail not visible in any public API. +A change in either extension's registered priority would break the ordering silently. + +**Python symlink traversal.** +``os.walk(followlinks=True)`` is used rather than ``Path.rglob()`` +because Python ≤ 3.12 does not follow symlinked directories in ``rglob``. +This is a known Python limitation, +but it means that any future refactor to use ``rglob`` would silently break auto-discovery in all combo builds. diff --git a/docs/concepts/index.rst b/docs/concepts/index.rst index 95dab276..b1f17ea0 100644 --- a/docs/concepts/index.rst +++ b/docs/concepts/index.rst @@ -10,3 +10,4 @@ Here you find explanations how and why docs-as-code works the way it does. bidirectional_traceability docs_deps + any_folder