Fix any_folder missing files issue

BUILD

@@ -38,6 +38,9 @@ docs(
         "//src:all_sources",
     ],
     source_dir = "docs",
+    source_dir_extras = [
+        "//src/extensions/docs:doc_sources_extra",
+    ],
 )
 
 cli_helper(

docs.bzl

@@ -120,7 +120,7 @@ def _missing_requirements(deps):
         fail(msg)
     fail("This case should be unreachable?!")
 
-def docs(source_dir = "docs", data = [], deps = [], scan_code = []):
+def docs(source_dir = "docs", data = [], deps = [], scan_code = [], source_dir_extras = []):
     """Creates all targets related to documentation.
 
     By using this function, you'll get any and all updates for documentation targets in one place.
@@ -130,6 +130,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []):
       data: Additional data files to include in the documentation build.
       deps: Additional dependencies for the documentation build.
       scan_code: List of code targets to scan for source code links.
+      source_dir_extras: Additional source targets outside of source_dir (any_folder)
     """
 
     call_path = native.package_name()
@@ -166,8 +167,9 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []):
             source_dir + "/**/*.json",
             source_dir + "/**/*.csv",
             source_dir + "/**/*.inc",
+            source_dir + "/conf.py",
             "more_docs/**/*.rst",
-        ], allow_empty = True),
+        ], allow_empty = True) + source_dir_extras,
         visibility = ["//visibility:public"],
     )
 
@@ -181,7 +183,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []):
         name = "docs",
         tags = ["cli_help=Build documentation:\nbazel run //:docs"],
         srcs = ["@score_docs_as_code//src:incremental.py"],
-        data = data + [":sourcelinks_json"],
+        data = data + source_dir_extras + [":sourcelinks_json"],
         deps = deps,
         env = {
             "SOURCE_DIRECTORY": source_dir,
@@ -195,7 +197,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []):
         name = "docs_combo",
         tags = ["cli_help=Build full documentation with all dependencies:\nbazel run //:docs_combo"],
         srcs = ["@score_docs_as_code//src:incremental.py"],
-        data = data_with_docs_sources + [":merged_sourcelinks"],
+        data = data_with_docs_sources + source_dir_extras + [":merged_sourcelinks"],
         deps = deps,
         env = {
             "SOURCE_DIRECTORY": source_dir,
@@ -215,7 +217,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []):
         name = "docs_link_check",
         tags = ["cli_help=Verify Links inside Documentation:\nbazel run //:link_check\n (Note: this could take a long time)"],
         srcs = ["@score_docs_as_code//src:incremental.py"],
-        data = data,
+        data = data + source_dir_extras,
         deps = deps,
         env = {
             "SOURCE_DIRECTORY": source_dir,
@@ -228,7 +230,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []):
         name = "docs_check",
         tags = ["cli_help=Verify documentation:\nbazel run //:docs_check"],
         srcs = ["@score_docs_as_code//src:incremental.py"],
-        data = data + [":sourcelinks_json"],
+        data = data + source_dir_extras + [":sourcelinks_json"],
         deps = deps,
         env = {
             "SOURCE_DIRECTORY": source_dir,
@@ -242,7 +244,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []):
         name = "live_preview",
         tags = ["cli_help=Live preview documentation in the browser:\nbazel run //:live_preview"],
         srcs = ["@score_docs_as_code//src:incremental.py"],
-        data = data + [":sourcelinks_json"],
+        data = data + source_dir_extras + [":sourcelinks_json"],
         deps = deps,
         env = {
             "SOURCE_DIRECTORY": source_dir,
@@ -256,7 +258,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []):
         name = "live_preview_combo_experimental",
         tags = ["cli_help=Live preview full documentation with all dependencies in the browser:\nbazel run //:live_preview_combo_experimental"],
         srcs = ["@score_docs_as_code//src:incremental.py"],
-        data = data_with_docs_sources + [":merged_sourcelinks"],
+        data = data_with_docs_sources + source_dir_extras + [":merged_sourcelinks"],
         deps = deps,
         env = {
             "SOURCE_DIRECTORY": source_dir,
@@ -272,7 +274,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []):
         venv_name = ".venv_docs",
         reqs = deps,
         # Add dependencies to ide_support, so esbonio has access to them.
-        data = data,
+        data = data + source_dir_extras,
     )
 
     sphinx_docs(

docs/concepts/any_folder.rst

@@ -0,0 +1,125 @@
+.. _any_folder_concept:
+
+Any Folder
+==========
+
+The goal is simple: developers should be able to place documentation files
+anywhere in the repository, not only in ``docs/``.
+A component developer writing ``src/my_component/`` naturally wants the component's docs
+to live next to the code in ``src/my_component/docs/``.
+
+Sphinx, however, only reads files inside its source directory (``confdir``,usually ``docs/``).
+The ``score_any_folder`` extension bridges this gap
+by creating temporary symlinks inside ``confdir`` that point to the external directories.
+Sphinx then discovers the files as if they were always there.
+
+The fundamental conflict: two build modes
+-----------------------------------------
+
+The difficulty is that docs-as-code supports two structurally different build modes
+that pull in opposite directions.
+
+**Live/incremental builds**
+(``bazel run :docs``, ``bazel run :live_preview``, Esbonio)
+run Sphinx directly on the developer's real workspace filesystem.
+Every file in the repository is accessible by its real path.
+Symlinks created by ``score_any_folder`` work naturally here.
+
+**Combo builds** (``bazel run :docs_combo``)
+aggregate documentation from multiple external repositories into a single Sphinx run.
+External modules are not present as real directories on the filesystem;
+they are mounted from Bazel runfiles by ``sphinx_collections``.
+Each module's file tree appears under
+``_collections/{module}/`` as a directory symlink into the runfiles.
+
+.. mermaid::
+
+   graph LR
+       subgraph a["Module A"]
+           LIVE[":live_preview"]
+           SRCS["docs/**"]
+           EXT["src/extensions/docs/**"]
+           CONFA["conf.py"]
+           LIVE --> CONFA
+           CONFA -.-> EXT
+       end
+
+       subgraph b["Module B"]
+           COMBO[":docs_combo"]
+           SRCS2["docs/**"]
+           CONFB["conf.py"]
+           COMBO --> CONFB
+       end
+
+       LIVE --> SRCS
+       COMBO --> SRCS
+       COMBO --> SRCS2
+
+The conflict: ``score_any_folder`` is designed to create symlinks based on
+the ``score_any_folder_mapping`` in the *active* ``conf.py``.
+In a combo build the active conf.py belongs to the aggregating project,
+not to any external module.
+Each mounted module has its own ``conf.py`` with its own mapping,
+but nothing would apply it —
+so the symlinks for those modules are never created, and their docs are broken.
+
+How the conflict is resolved
+-----------------------------
+
+The resolution has three cooperating parts.
+
+#. **Runtime symlinks (score_any_folder).**
+   ``score_any_folder`` runs at Sphinx event priority 600,
+   after ``sphinx_collections`` (priority 500) has already created all mounts.
+   It then scans ``confdir`` for ``conf.py`` files in *subdirectories*,
+   which includes the mounted modules' conf files.
+   For each one it finds, it extracts the ``score_any_folder_mapping``
+   and applies the symlink mapping relative to that module's directory —
+   exactly as if Sphinx were building that module standalone.
+
+#. **Deduplication (exclude_patterns).**
+   ``sphinx_collections`` mounts a module's *entire* runfiles tree, not just its ``docs/`` directory.
+   This means files from ``source_dir_extras`` (e.g. ``src/extensions/docs/``) are visible to Sphinx at two paths:
+   directly through the mount *and* through the symlink just created.
+   To prevent duplicate-label errors,
+   ``score_any_folder`` adds the direct paths to Sphinx's ``exclude_patterns``
+   so only the symlinked path is indexed.
+
+#. **Bazel dependency declarations (source_dir_extras).**
+   For hermetic ``needs_json`` builds, Bazel must know about every input file before the build starts.
+   Files reachable only through runtime symlinks are invisible to Bazel's dependency analysis.
+   The ``source_dir_extras`` parameter of ``docs()`` lets a module declare these external directories
+   as explicit Bazel filegroup targets.
+   For combo builds, ``docs_sources`` also includes ``conf.py`` itself
+   so that the auto-discovery scan in step 1 can find and read it from within the mounted runfiles.
+
+Architectural risks
+--------------------
+
+This design works today but rests on several assumptions that could break silently.
+
+**Symlinks into runfiles.**
+Secondary symlinks are created inside the Bazel runfiles tree
+(because the sphinx_collections mount is itself a symlink into it).
+This works because ``bazel run`` currently provides a writable runfiles tree.
+If Bazel ever makes the runfiles read-only, symlink creation will fail.
+
+**exclude_patterns is a workaround for a mount granularity problem.**
+The real cause of the duplication is that ``sphinx_collections`` mounts the full module root, not just ``docs/``.
+The ``exclude_patterns`` approach is a compensating hack.
+If Sphinx changes how it processes ``exclude_patterns``,
+or if ``sphinx_collections`` changes how it creates mounts,
+the deduplication can silently stop working.
+The symptom would be a flood of duplicate-label warnings in combo builds.
+
+**Timing dependency.**
+The entire secondary-scan mechanism depends on ``sphinx_collections``
+completing its mounts before ``score_any_folder`` scans for conf.py files.
+This is enforced by Sphinx event priority (600 vs. 500), an internal detail not visible in any public API.
+A change in either extension's registered priority would break the ordering silently.
+
+**Python symlink traversal.**
+``os.walk(followlinks=True)`` is used rather than ``Path.rglob()``
+because Python ≤ 3.12 does not follow symlinked directories in ``rglob``.
+This is a known Python limitation,
+but it means that any future refactor to use ``rglob`` would silently break auto-discovery in all combo builds.

docs/concepts/index.rst

@@ -10,3 +10,4 @@ Here you find explanations how and why docs-as-code works the way it does.
 
    bidirectional_traceability
    docs_deps
+   any_folder

docs/how-to/any_folder.rst

@@ -12,12 +12,14 @@ You can symlink the folders by adding to your ``conf.py``:
        "../score/containers/docs": "component/containers",
    }
 
-With this configuration, all files in ``score/containers/docs/`` become available at ``docs/component/containers/``.
+All files in ``score/containers/docs/`` become available at ``docs/component/containers/``.
+Include them via ``toctree`` as usual.
 
 If you have ``docs/component/overview.rst``, for example,
 you can include the component documentation via ``toctree``:
 
 .. code-block:: rst
+   :caption: some rst file
 
    .. toctree::
 
@@ -27,3 +29,21 @@ Only relative links are allowed.
 
 The symlinks will show up in your sources.
 **Don't commit the symlinks to git!**
+
+Bazel
+-----
+
+When building with Bazel, declare the mapped directories as ``source_dir_extras``
+in your ``docs()`` call so Bazel tracks them as dependencies:
+
+.. code-block:: python
+   :caption: BUILD
+
+   docs(
+       source_dir = "docs",
+       source_dir_extras = ["//score/containers:docs_sources"],
+       ...
+   )
+
+This is necessary for sandboxed builds.
+For example, when other modules use your documentation's ``needs.json`` as a dependency.

src/extensions/docs/BUILD

@@ -0,0 +1,5 @@
+filegroup(
+    name = "doc_sources_extra",
+    srcs = ["data_flow.png"] + glob(["*.rst", "*.md"]),
+    visibility = ["//visibility:public"],
+)

src/extensions/docs/any_folder.rst

@@ -24,8 +24,23 @@ Sphinx then discovers and buildsthose files as if they were part of ``docs/`` fr
 The extension hooks into the ``builder-inited`` event,
 which fires before Sphinx reads any documents.
 
+Configuration reference
+-----------------------
+
+``score_any_folder_mapping``
+    *dict[str, str]*, default ``{}``
+
+    Maps source directories to symlink paths, both relative to ``confdir``.
+    Applied on every Sphinx build.
+
+    .. code-block:: python
+
+       score_any_folder_mapping = {
+           "../src/my_module/docs": "my_module",
+       }
+
 Difference to Sphinx-Collections
---------------------------------
+---------------------------------
 
 The extension `sphinx-collections <https://sphinx-collections.readthedocs.io/>`_
 is very similar to this extension.