From bde02c22275adf0c22d6ba6ff8a82acb01adfef4 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Wed, 11 Mar 2026 20:48:26 +0100 Subject: [PATCH 01/34] WIP: Local Works somewhat --- docs.bzl | 17 +- scripts_bazel/generate_sourcelinks_cli.py | 51 ++++- scripts_bazel/merge_sourcelinks.py | 70 ++++++- .../score_source_code_linker/__init__.py | 184 +++++++++--------- .../score_source_code_linker/metadata.py | 23 +++ .../module_source_links.py | 137 +++++++++++++ .../need_source_links.py | 51 +++++ .../score_source_code_linker/needlinks.py | 114 +++++++++-- src/helper_lib/additional_functions.py | 25 +++ 9 files changed, 555 insertions(+), 117 deletions(-) create mode 100644 src/extensions/score_source_code_linker/metadata.py create mode 100644 src/extensions/score_source_code_linker/module_source_links.py diff --git a/docs.bzl b/docs.bzl index cd41a4c83..fe26590e8 100644 --- a/docs.bzl +++ b/docs.bzl @@ -69,7 +69,7 @@ def _rewrite_needs_json_to_sourcelinks(labels): out.append(s) return out -def _merge_sourcelinks(name, sourcelinks): +def _merge_sourcelinks(name, sourcelinks, known_good = None): """Merge multiple sourcelinks JSON files into a single file. Args: @@ -77,15 +77,22 @@ def _merge_sourcelinks(name, sourcelinks): sourcelinks: List of sourcelinks JSON file targets """ + extra_srcs = [] + known_good_arg = "" + if known_good != None: + extra_srcs = [known_good] + known_good_arg = "--known_good $(location %s)" % known_good + native.genrule( name = name, - srcs = sourcelinks, + srcs = sourcelinks + extra_srcs, outs = [name + ".json"], cmd = """ $(location @score_docs_as_code//scripts_bazel:merge_sourcelinks) \ --output $@ \ + {known_good_arg} \ $(SRCS) - """, + """.format(known_good_arg = known_good_arg), tools = ["@score_docs_as_code//scripts_bazel:merge_sourcelinks"], ) @@ -120,7 +127,7 @@ def _missing_requirements(deps): fail(msg) fail("This case should be unreachable?!") -def docs(source_dir = "docs", data = [], deps = [], scan_code = []): +def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good = None): """Creates all targets related to documentation. By using this function, you'll get any and all updates for documentation targets in one place. @@ -175,7 +182,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []): data_with_docs_sources = _rewrite_needs_json_to_docs_sources(data) additional_combo_sourcelinks = _rewrite_needs_json_to_sourcelinks(data) - _merge_sourcelinks(name = "merged_sourcelinks", sourcelinks = [":sourcelinks_json"] + additional_combo_sourcelinks) + _merge_sourcelinks(name = "merged_sourcelinks", sourcelinks = [":sourcelinks_json"] + additional_combo_sourcelinks, known_good = known_good) py_binary( name = "docs", diff --git a/scripts_bazel/generate_sourcelinks_cli.py b/scripts_bazel/generate_sourcelinks_cli.py index 4291b97c5..27a0e4508 100644 --- a/scripts_bazel/generate_sourcelinks_cli.py +++ b/scripts_bazel/generate_sourcelinks_cli.py @@ -26,24 +26,51 @@ _extract_references_from_file, # pyright: ignore[reportPrivateUsage] TODO: move it out of the extension and into this script ) from src.extensions.score_source_code_linker.needlinks import ( - store_source_code_links_json, + store_source_code_links_with_metadata_json, ) +from src.extensions.score_source_code_linker.metadata import MetaData logging.basicConfig(level=logging.INFO, format="%(message)s") logger = logging.getLogger(__name__) +def parse_module_name_from_path(path: Path) -> str: + """ + Parse out the Module-Name from the filename gotten + /home/user/.cache/bazel/aksj37981712/external/score_docs_as_code+/src/tests/testfile.py + => score_docs_as_code + """ + + # COMBO BUILD + # If external is in the filepath that gets parsed => + # file is in an external module => combo build + # e.g. .../external/score_docs_as_code+/src/helper_lib/__init__.py + + # PATH if we are in local repository + # PosixPath('src/helper_lib/test_helper_lib.py') + # Path if we are in combo build and externally + # PosixPath('external/score_docs_as_code+/src/helper_lib/test_helper_lib.py' + print("======== THIs IS PATH we PARSIGN FOr MODULE NAME") + print(path) + if str(path).startswith("external/"): + module_raw = str(path).removeprefix("external/") + filepath_split = str(module_raw).split("/", maxsplit=1) + module_name = str(filepath_split[0].removesuffix("+")) + return module_name + return "local_module" + + def main(): parser = argparse.ArgumentParser( description="Generate source code links JSON from source files" ) - parser.add_argument( + _ = parser.add_argument( "--output", required=True, type=Path, help="Output JSON file path", ) - parser.add_argument( + _ = parser.add_argument( "files", nargs="*", type=Path, @@ -53,15 +80,29 @@ def main(): args = parser.parse_args() all_need_references = [] + metadata: MetaData = { + "module_name": "", + "hash": "", + "url": "", + } + metadata_set = False for file_path in args.files: + if "known_good.json" not in str(file_path) and not metadata_set: + metadata["module_name"] = parse_module_name_from_path(file_path) + print("================") + print(metadata) + print("===============") + print("METADATA SET") + metadata_set = True abs_file_path = file_path.resolve() assert abs_file_path.exists(), abs_file_path references = _extract_references_from_file( abs_file_path.parent, Path(abs_file_path.name) ) all_need_references.extend(references) - - store_source_code_links_json(args.output, all_need_references) + store_source_code_links_with_metadata_json( + file=args.output, metadata=metadata, needlist=all_need_references + ) logger.info( f"Found {len(all_need_references)} need references in {len(args.files)} files" ) diff --git a/scripts_bazel/merge_sourcelinks.py b/scripts_bazel/merge_sourcelinks.py index f194e19ca..034d45db5 100644 --- a/scripts_bazel/merge_sourcelinks.py +++ b/scripts_bazel/merge_sourcelinks.py @@ -20,22 +20,71 @@ import logging import sys from pathlib import Path +from typing import Any + +# from src.extensions.score_source_code_linker.need_source_links import ( +# store_source_code_links_combined_json, +# ) logging.basicConfig(level=logging.INFO, format="%(message)s") logger = logging.getLogger(__name__) +# [ +# PosixPath('bazel-out/k8-fastbuild/bin/sourcelinks_json.json'), +# PosixPath('bazel-out/k8-fastbuild/bin/external/score_persistency+/sourcelinks_json.json'), +# PosixPath('bazel-out/k8-fastbuild/bin/external/score_orchestrator+/sourcelinks_json.json'), +# PosixPath('bazel-out/k8-fastbuild/bin/external/score_kyron+/sourcelinks_json.json'), +# PosixPath('bazel-out/k8-fastbuild/bin/external/score_baselibs+/sourcelinks_json.json'), +# PosixPath('bazel-out/k8-fastbuild/bin/external/score_baselibs_rust+/sourcelinks_json.json'), +# PosixPath('bazel-out/k8-fastbuild/bin/external/score_logging+/sourcelinks_json.json'), +# PosixPath('bazel-out/k8-fastbuild/bin/external/score_platform+/sourcelinks_json.json'), +# PosixPath('bazel-out/k8-fastbuild/bin/external/score_process+/sourcelinks_json.json'), +# PosixPath('bazel-out/k8-fastbuild/bin/external/score_docs_as_code+/sourcelinks_json.json') +# ] + + +""" +if bazel-out/k8-fastbuild/bin/external/ in file_path => module is external +otherwise it's local +if local => module_name & hash == empty +if external => parse thing for module_name => look up known_good json for hash & url +""" + + +def parse_info_from_known_good( + known_good_json: Path, module_name: str +) -> tuple[str, str]: + print("===THIS IS MODULE NAME WE LOOK FOR===========") + print(module_name) + with open(known_good_json, "r") as f: + kg_json = json.load(f) + for category in kg_json["modules"].values(): + print("===THIS IS CATEGORY=========") + print(category) + if module_name in category: + print("===THIS IS MODULE NAME INSIDe CATEGORY===========") + print(module_name) + m = category[module_name] + return (m["hash"], m["repo"].removesuffix(".git")) + raise KeyError(f"Module {module_name!r} not found in known_good_json.") + def main(): parser = argparse.ArgumentParser( description="Merge multiple sourcelinks JSON files into one" ) - parser.add_argument( + _ = parser.add_argument( "--output", required=True, type=Path, help="Output merged JSON file path", ) - parser.add_argument( + _ = parser.add_argument( + "--known_good", + default=None, + help="Optional path to a 'known good' JSON file (provided by Bazel).", + ) + _ = parser.add_argument( "files", nargs="*", type=Path, @@ -43,13 +92,26 @@ def main(): ) args = parser.parse_args() + all_files = [x for x in args.files if "known_good.json" not in str(x)] merged = [] - for json_file in args.files: + for json_file in all_files: with open(json_file) as f: data = json.load(f) + metadata = data[0] + if metadata["module_name"] and metadata["module_name"] != "local_module": + hash, repo = parse_info_from_known_good( + known_good_json=args.known_good, module_name=metadata["module_name"] + ) + metadata["hash"] = hash + metadata["url"] = repo + # In the case that 'metadata[module_name]' is empty + # hash & url are already existing and empty inside of 'metadata' + # Therefore all 3 keys will be written to needlinks in each branch + for d in data[1:]: + d.update(metadata) assert isinstance(data, list), repr(data) - merged.extend(data) + merged.extend(data[1:]) with open(args.output, "w") as f: json.dump(merged, f, indent=2, ensure_ascii=False) diff --git a/src/extensions/score_source_code_linker/__init__.py b/src/extensions/score_source_code_linker/__init__.py index cf9843dc5..7e0ff0252 100644 --- a/src/extensions/score_source_code_linker/__init__.py +++ b/src/extensions/score_source_code_linker/__init__.py @@ -21,7 +21,6 @@ # This whole directory implements the above mentioned tool requirements import os -from collections import defaultdict from copy import deepcopy from pathlib import Path from typing import cast @@ -40,10 +39,19 @@ SourceCodeLinks, load_source_code_links_combined_json, store_source_code_links_combined_json, + group_by_need, ) + +from src.extensions.score_source_code_linker.module_source_links import ( + group_needs_by_module, + store_module_source_links_json, + load_module_source_links_json, +) + from src.extensions.score_source_code_linker.needlinks import ( NeedLink, load_source_code_links_json, + load_source_code_links_with_metadata_json, ) from src.extensions.score_source_code_linker.testlink import ( DataForTestLink, @@ -71,53 +79,6 @@ # ╰──────────────────────────────────────╯ -def group_by_need( - source_code_links: list[NeedLink], - test_case_links: list[DataForTestLink] | None = None, -) -> list[SourceCodeLinks]: - """ - Groups the given need links and test case links by their need ID. - Returns a nested dictionary structure with 'CodeLink' and 'TestLink' categories. - Example output: - - - { - "need": "", - "links": { - "CodeLinks": [NeedLink, NeedLink, ...], - "TestLinks": [testlink, testlink, ...] - } - } - """ - # TODO: I wonder if there is a more efficent way to do this - grouped_by_need: dict[str, NeedSourceLinks] = defaultdict( - lambda: NeedSourceLinks(TestLinks=[], CodeLinks=[]) - ) - - # Group source code links - for needlink in source_code_links: - grouped_by_need[needlink.need].CodeLinks.append(needlink) - - # Group test case links - if test_case_links is not None: - for testlink in test_case_links: - grouped_by_need[testlink.need].TestLinks.append(testlink) - - # Build final list of SourceCodeLinks - result: list[SourceCodeLinks] = [ - SourceCodeLinks( - need=need, - links=NeedSourceLinks( - CodeLinks=need_links.CodeLinks, - TestLinks=need_links.TestLinks, - ), - ) - for need, need_links in grouped_by_need.items() - ] - - return result - - def get_cache_filename(build_dir: Path, filename: str) -> Path: """ Returns the path to the cache file for the source code linker. @@ -142,14 +103,19 @@ def build_and_save_combined_file(outdir: Path): else: source_code_links_json = Path(source_code_links_json) - source_code_links = load_source_code_links_json(source_code_links_json) + # This isn't pretty will think of a better solution later, for now this should work + try: + source_code_links = load_source_code_links_json(source_code_links_json) + except AssertionError: + source_code_links = load_source_code_links_with_metadata_json( + source_code_links_json + ) test_code_links = load_test_xml_parsed_json( get_cache_filename(outdir, "score_xml_parser_cache.json") ) - + scl_list = group_by_need(source_code_links, test_code_links) store_source_code_links_combined_json( - outdir / "score_scl_grouped_cache.json", - group_by_need(source_code_links, test_code_links), + outdir / "score_scl_grouped_cache.json", scl_list ) @@ -254,10 +220,10 @@ def setup_test_code_linker(app: Sphinx, env: BuildEnvironment): def register_combined_linker(app: Sphinx): - # Registering the combined linker to Sphinx + # Registering the final combine linker to Sphinx # priority is set to make sure it is called in the right order. - # Needs to be called after xml parsing & codelink - app.connect("env-updated", setup_combined_linker, priority=507) + # Needs to be called after xml parsing & codelink & combined_linker + app.connect("env-updated", setup_combined_linker, priority=510) def setup_combined_linker(app: Sphinx, _: BuildEnvironment): @@ -272,6 +238,37 @@ def setup_combined_linker(app: Sphinx, _: BuildEnvironment): build_and_save_combined_file(app.outdir) +def register_module_linker(app: Sphinx): + # Registering the combined linker to Sphinx + # priority is set to make sure it is called in the right order. + # Needs to be called after xml parsing & codelink + app.connect("env-updated", setup_module_linker, priority=520) + + +def build_and_save_module_scl_file(outdir: Path): + scl_links = load_source_code_links_combined_json( + get_cache_filename(outdir, "score_scl_grouped_cache.json") + ) + mcl_links = group_needs_by_module(scl_links) + store_module_source_links_json( + outdir / "score_module_grouped_scl_cache.json", mcl_links + ) + + +def setup_module_linker(app: Sphinx, _: BuildEnvironment): + grouped_cache = get_cache_filename( + app.outdir, "score_module_grouped_scl_cache.json" + ) + gruped_cache_exists = grouped_cache.exists() + # TODO this cache should be done via Bazel + if not gruped_cache_exists or not app.config.skip_rescanning_via_source_code_linker: + LOGGER.debug( + "Did not find combined json 'score_module_grouped_scl_cache.json' " + "in _build. Generating new one" + ) + build_and_save_module_scl_file(app.outdir) + + def setup_once(app: Sphinx): # might be the only way to solve this? if "skip_rescanning_via_source_code_linker" in app.config: @@ -295,9 +292,10 @@ def setup_once(app: Sphinx): setup_source_code_linker(app, ws_root) register_test_code_linker(app) register_combined_linker(app) + register_module_linker(app) - # Priorty=510 to ensure it's called after the test code linker & combined connection - app.connect("env-updated", inject_links_into_needs, priority=510) + # Priorty=515 to ensure it's called after the test code linker & combined connection + app.connect("env-updated", inject_links_into_needs, priority=525) def setup(app: Sphinx) -> dict[str, str | bool]: @@ -352,42 +350,46 @@ def inject_links_into_needs(app: Sphinx, env: BuildEnvironment) -> None: f"?? Need {id} already has testlink: {need.get('testlink')}" ) - source_code_links_by_need = load_source_code_links_combined_json( - get_cache_filename(app.outdir, "score_scl_grouped_cache.json") + scl_by_module = load_module_source_links_json( + get_cache_filename(app.outdir, "score_module_grouped_scl_cache.json") ) + # source_code_links_by_need = load_source_code_links_combined_json( + # get_cache_filename(app.outdir, "score_scl_grouped_cache.json") + # ) + for module_grouped_needs in scl_by_module: + for source_code_links in module_grouped_needs.needs: + need = find_need(needs_copy, source_code_links.need) + if need is None: + # TODO: print github annotations as in https://github.com/eclipse-score/bazel_registry/blob/7423b9996a45dd0a9ec868e06a970330ee71cf4f/tools/verify_semver_compatibility_level.py#L126-L129 + for n in source_code_links.links.CodeLinks: + LOGGER.warning( + f"{n.file}:{n.line}: Could not find {source_code_links.need} " + "in documentation [CODE LINK]", + type="score_source_code_linker", + ) + for n in source_code_links.links.TestLinks: + LOGGER.warning( + f"{n.file}:{n.line}: Could not find {source_code_links.need} " + "in documentation [TEST LINK]", + type="score_source_code_linker", + ) + continue + + need_as_dict = cast(dict[str, object], need) + metadata = module_grouped_needs.module_name + need_as_dict["source_code_link"] = ", ".join( + f"{get_github_link(metadata, n)}<>{n.file}:{n.line}" + for n in source_code_links.links.CodeLinks + ) + need_as_dict["testlink"] = ", ".join( + f"{get_github_link(metadata, n)}<>{n.name}" + for n in source_code_links.links.TestLinks + ) - for source_code_links in source_code_links_by_need: - need = find_need(needs_copy, source_code_links.need) - if need is None: - # TODO: print github annotations as in https://github.com/eclipse-score/bazel_registry/blob/7423b9996a45dd0a9ec868e06a970330ee71cf4f/tools/verify_semver_compatibility_level.py#L126-L129 - for n in source_code_links.links.CodeLinks: - LOGGER.warning( - f"{n.file}:{n.line}: Could not find {source_code_links.need} " - "in documentation [CODE LINK]", - type="score_source_code_linker", - ) - for n in source_code_links.links.TestLinks: - LOGGER.warning( - f"{n.file}:{n.line}: Could not find {source_code_links.need} " - "in documentation [TEST LINK]", - type="score_source_code_linker", - ) - continue - - need_as_dict = cast(dict[str, object], need) - - need_as_dict["source_code_link"] = ", ".join( - f"{get_github_link(n)}<>{n.file}:{n.line}" - for n in source_code_links.links.CodeLinks - ) - need_as_dict["testlink"] = ", ".join( - f"{get_github_link(n)}<>{n.name}" for n in source_code_links.links.TestLinks - ) - - # NOTE: Removing & adding the need is important to make sure - # the needs gets 're-evaluated'. - Needs_Data.remove_need(need["id"]) - Needs_Data.add_need(need) + # NOTE: Removing & adding the need is important to make sure + # the needs gets 're-evaluated'. + Needs_Data.remove_need(need["id"]) + Needs_Data.add_need(need) # ╭──────────────────────────────────────╮ diff --git a/src/extensions/score_source_code_linker/metadata.py b/src/extensions/score_source_code_linker/metadata.py new file mode 100644 index 000000000..03acc537a --- /dev/null +++ b/src/extensions/score_source_code_linker/metadata.py @@ -0,0 +1,23 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +from typing import TypedDict, TypeGuard + +class MetaData(TypedDict): + module_name: str + hash: str + url: str + +def is_metadata(x: object) -> TypeGuard[MetaData]: + # Make this as strict/loose as you want; at minimum, it must be a dict. + return isinstance(x, dict) and {"module_name", "hash", "url"} <= x.keys() diff --git a/src/extensions/score_source_code_linker/module_source_links.py b/src/extensions/score_source_code_linker/module_source_links.py new file mode 100644 index 000000000..3957733f0 --- /dev/null +++ b/src/extensions/score_source_code_linker/module_source_links.py @@ -0,0 +1,137 @@ +# ******************************************************************************* +# Copyright (c) 2025 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + + +from dataclasses import dataclass, asdict, field +import json +from typing import Any +from pathlib import Path + + +from src.extensions.score_source_code_linker.need_source_links import ( + NeedSourceLinks, + SourceCodeLinks, + SourceCodeLinks_JSON_Decoder, +) +from src.extensions.score_source_code_linker.needlinks import NeedLink +from src.extensions.score_source_code_linker.testlink import DataForTestLink + + +@dataclass +class moduleInfo: + name: str + hash: str + url: str + + +@dataclass +class ModuleSourceLinks: + module_name: moduleInfo + needs: list[SourceCodeLinks] = field(default_factory=list) + + +class ModuleSourceLinks_JSON_Encoder(json.JSONEncoder): + def default(self, o: object): + if isinstance(o, Path): + return str(o) + # We do not want to save the metadata inside the codelink + # (hash, module_name, url) + if isinstance(o, NeedLink): + return o.to_dict_without_metadata() + if isinstance( + o, ModuleSourceLinks | SourceCodeLinks | DataForTestLink | NeedSourceLinks + ): + return asdict(o) + return super().default(o) + + +def ModuleSourceLinks_JSON_Decoder( + d: dict[str, Any], +) -> ModuleSourceLinks | dict[str, Any]: + if "module_name" in d and "needs" in d: + module_name = d["module_name"] + needs = d["needs"] + return ModuleSourceLinks( + module_name=moduleInfo( + name=module_name.get("module_name"), + hash=module_name.get("hash"), + url=module_name.get("url"), + ), + # We know this can only be list[SourceCodeLinks] and nothing else + # Therefore => we ignore the type error here + needs=[SourceCodeLinks_JSON_Decoder(need) for need in needs] # type: ignore + ) + return d + + +def store_module_source_links_json( + file: Path, source_code_links: list[ModuleSourceLinks] +): + # After `rm -rf _build` or on clean builds the directory does not exist, so we need + # to create it + file.parent.mkdir(exist_ok=True) + with open(file, "w") as f: + json.dump( + source_code_links, + f, + cls=ModuleSourceLinks_JSON_Encoder, + indent=2, + ensure_ascii=False, + ) + + +def load_module_source_links_json(file: Path) -> list[ModuleSourceLinks]: + links: list[ModuleSourceLinks] = json.loads( + file.read_text(encoding="utf-8"), + object_hook=ModuleSourceLinks_JSON_Decoder, + ) + assert isinstance(links, list), ( + "The combined source code linker links should be " + "a list of SourceCodeLinks objects." + ) + assert all(isinstance(link, ModuleSourceLinks) for link in links), ( + "All items in combined_source_code_linker_cache should be " + "SourceCodeLinks objects." + ) + return links + + +def group_needs_by_module(links: list[SourceCodeLinks]) -> list[ModuleSourceLinks]: + module_groups: dict[str, ModuleSourceLinks] = {} + + for source_link in links: + if not source_link.links.CodeLinks: + continue + + first_link = source_link.links.CodeLinks[0] + module_key = first_link.module_name + + if module_key not in module_groups: + module_groups[module_key] = ModuleSourceLinks( + module_name=moduleInfo(name=module_key, hash=first_link.hash, url=first_link.url) + ) + + module_groups[module_key].needs.append(source_link) # Much clearer! + + return [ + ModuleSourceLinks(module_name=group.module_name, needs=group.needs) + for group in module_groups.values() + ] + + +# # Pouplate Metadata +# # Since all metadata inside the Codelinks is the same +# # we can just arbitrarily grab the first one +# module_name=need_links.CodeLinks[0].module_name, +# hash=need_links.CodeLinks[0].hash, +# url=need_links.CodeLinks[0].url, diff --git a/src/extensions/score_source_code_linker/need_source_links.py b/src/extensions/score_source_code_linker/need_source_links.py index 6c738da8e..2deb4c5e8 100644 --- a/src/extensions/score_source_code_linker/need_source_links.py +++ b/src/extensions/score_source_code_linker/need_source_links.py @@ -21,6 +21,7 @@ import json from dataclasses import asdict, dataclass, field +from collections import defaultdict from pathlib import Path from typing import Any @@ -108,3 +109,53 @@ def load_source_code_links_combined_json(file: Path) -> list[SourceCodeLinks]: "SourceCodeLinks objects." ) return links + + +def group_by_need( + source_code_links: list[NeedLink], + test_case_links: list[DataForTestLink] | None = None, +) -> list[SourceCodeLinks]: + """ + Groups the given need links and test case links by their need ID. + Returns a nested dictionary structure with 'CodeLink' and 'TestLink' categories. + Example output: + + + { + "need": "", + "module_name": , + "hash": , + "url": , + "links": { + "CodeLinks": [NeedLink, NeedLink, ...], + "TestLinks": [testlink, testlink, ...] + } + } + """ + # TODO: I wonder if there is a more efficent way to do this + grouped_by_need: dict[str, NeedSourceLinks] = defaultdict( + lambda: NeedSourceLinks(TestLinks=[], CodeLinks=[]) + ) + + # Group source code links + for needlink in source_code_links: + grouped_by_need[needlink.need].CodeLinks.append(needlink) + + # Group test case links + if test_case_links is not None: + for testlink in test_case_links: + grouped_by_need[testlink.need].TestLinks.append(testlink) + + # Build final list of SourceCodeLinks + result: list[SourceCodeLinks] = [ + SourceCodeLinks( + need=need, + links=NeedSourceLinks( + CodeLinks=need_links.CodeLinks, + TestLinks=need_links.TestLinks, + ), + ) + for need, need_links in grouped_by_need.items() + ] + + return result diff --git a/src/extensions/score_source_code_linker/needlinks.py b/src/extensions/score_source_code_linker/needlinks.py index 348147292..a727025bd 100644 --- a/src/extensions/score_source_code_linker/needlinks.py +++ b/src/extensions/score_source_code_linker/needlinks.py @@ -16,10 +16,11 @@ import os from dataclasses import asdict, dataclass from pathlib import Path -from typing import Any +from typing import Any, TypeGuard +from src.extensions.score_source_code_linker.metadata import MetaData, is_metadata -@dataclass(frozen=True, order=True) +@dataclass(order=True) class NeedLink: """Represents a single template string finding in a file.""" @@ -28,6 +29,19 @@ class NeedLink: tag: str need: str full_line: str + module_name: str = "" + hash: str = "" + url: str = "" + + def to_dict_full(self) -> dict[str, str | Path]: + return asdict(self) + + def to_dict_without_metadata(self) -> dict[str, str | Path]: + d = asdict(self) + d.pop("module_name", None) + d.pop("hash", None) + d.pop("url", None) + return d def DefaultNeedLink() -> NeedLink: @@ -41,6 +55,8 @@ def DefaultNeedLink() -> NeedLink: tag="", need="", full_line="", + # Module_name, hash, url are defaulted to "" + # therefore not needed to be listed ) @@ -61,26 +77,99 @@ def needlink_decoder(d: dict[str, Any]) -> NeedLink | dict[str, Any]: tag=d["tag"], need=d["need"], full_line=d["full_line"], + module_name=d.get("module_name", ""), + hash=d.get("hash", ""), + url=d.get("url", ""), ) # It's something else, pass it on to other decoders return d -def store_source_code_links_json(file: Path, needlist: list[NeedLink]): - # After `rm -rf _build` or on clean builds the directory does not exist, - # so we need to create it +def store_source_code_links_with_metadata_json( + file: Path, metadata: MetaData, needlist: list[NeedLink] +) -> None: + """ + Writes a JSON array: + [ meta_dict, needlink1, needlink2, ... ] + + meta_dict must include: + module_name, hash, url + """ + payload: list[object] = [metadata, *needlist] + file.parent.mkdir(exist_ok=True) - with open(file, "w") as f: - json.dump( - needlist, - f, - cls=NeedLinkEncoder, # use your custom encoder - indent=2, - ensure_ascii=False, + with open(file, "w", encoding="utf-8") as f: + json.dump(payload, f, cls=NeedLinkEncoder, indent=2, ensure_ascii=False) + + +def store_source_code_links_json(file: Path, needlist: list[NeedLink]) -> None: + """ + Writes a JSON array: + [ meta_dict, needlink1, needlink2, ... ] + + meta_dict must include: + module_name, hash, url + """ + + file.parent.mkdir(exist_ok=True) + with open(file, "w", encoding="utf-8") as f: + json.dump(needlist, f, cls=NeedLinkEncoder, indent=2, ensure_ascii=False) + + +def _is_needlink_list(xs: list[object]) -> TypeGuard[list[NeedLink]]: + return all(isinstance(link, NeedLink) for link in xs) + + +def load_source_code_links_with_metadata_json(file: Path) -> list[NeedLink]: + """ + Expects the JSON array where first is a meta_dict: + [ meta_dict, needlink1, needlink2, ... ] + Returns: + [NeedLink, NeedLink, ...] + + This normally should be the one called 'locally' => :docs target + """ + if not file.is_absolute(): + ws_root = os.environ.get("BUILD_WORKSPACE_DIRECTORY") + if ws_root: + file = Path(ws_root) / file + + data: list[object] = json.loads( + file.read_text(encoding="utf-8"), + object_hook=needlink_decoder, + ) + links: list[object] = [] + if not is_metadata(data[0]): + raise TypeError( + "If you do not have a 'metadata' dict as the first one in the json " + "you might wanted to call the load without metadata named: " + "'load_source_code_links_json'" ) + metadata: MetaData = data[0] + links = data[1:] + if not _is_needlink_list(links): + raise TypeError( + "In local build context all items after" + f"metadata must decode to NeedLink objects. File: {file}" + ) + for d in links: + d.module_name = metadata["module_name"] + d.hash = metadata["hash"] + d.url = metadata["url"] + return links def load_source_code_links_json(file: Path) -> list[NeedLink]: + """ + Expects the JSON array with needlinks + *that already have extra info in them* (module_name, hash, url): + [ needlink1, needlink2, ... ] + Returns: + [NeedLink, NeedLink, ...] + + This normally should be the one called in combo builds + => :docs_combo_experimental target + """ if not file.is_absolute(): # use env variable set by Bazel ws_root = os.environ.get("BUILD_WORKSPACE_DIRECTORY") @@ -98,3 +187,4 @@ def load_source_code_links_json(file: Path) -> list[NeedLink]: "All items in source_code_links should be NeedLink objects." ) return links + diff --git a/src/helper_lib/additional_functions.py b/src/helper_lib/additional_functions.py index 5b1ce6d98..872d66af2 100644 --- a/src/helper_lib/additional_functions.py +++ b/src/helper_lib/additional_functions.py @@ -14,6 +14,7 @@ # Import types that depend on score_source_code_linker from src.extensions.score_source_code_linker.needlinks import DefaultNeedLink, NeedLink +from src.extensions.score_source_code_linker.module_source_links import moduleInfo from src.extensions.score_source_code_linker.testlink import ( DataForTestLink, DataOfTestCase, @@ -26,6 +27,19 @@ def get_github_link( + metadata: moduleInfo, + link: NeedLink | DataForTestLink | DataOfTestCase | None = None, +) -> str: + if link is None: + link = DefaultNeedLink() + if not metadata.hash: + # Local path (//:docs) + return get_github_link_from_git(link) + # Ref-Integration path (//:docs_combo..) + return get_github_link_from_json(metadata, link) + + +def get_github_link_from_git( link: NeedLink | DataForTestLink | DataOfTestCase | None = None, ) -> str: if link is None: @@ -36,3 +50,14 @@ def get_github_link( base_url = get_github_base_url() current_hash = get_current_git_hash(passed_git_root) return f"{base_url}/blob/{current_hash}/{link.file}#L{link.line}" + + +def get_github_link_from_json( + metadata: moduleInfo, + link: NeedLink | DataForTestLink | DataOfTestCase | None = None, +): + if link is None: + link = DefaultNeedLink() + base_url = metadata.url + current_hash = metadata.hash + return f"{base_url}/blob/{current_hash}/{link.file}#L{link.line}" From a898eee3194806d5030e24775ba7b4069fae23c8 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Wed, 11 Mar 2026 21:55:42 +0100 Subject: [PATCH 02/34] WIP: seems to work in ref & local --- scripts_bazel/generate_sourcelinks_cli.py | 13 ++++++++----- .../generate_source_code_links_json.py | 19 +++++++++++-------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/scripts_bazel/generate_sourcelinks_cli.py b/scripts_bazel/generate_sourcelinks_cli.py index 27a0e4508..2f26784ff 100644 --- a/scripts_bazel/generate_sourcelinks_cli.py +++ b/scripts_bazel/generate_sourcelinks_cli.py @@ -89,15 +89,18 @@ def main(): for file_path in args.files: if "known_good.json" not in str(file_path) and not metadata_set: metadata["module_name"] = parse_module_name_from_path(file_path) - print("================") - print(metadata) - print("===============") - print("METADATA SET") + # print("================") + # print(metadata) + # print("===============") + # print("METADATA SET") metadata_set = True abs_file_path = file_path.resolve() assert abs_file_path.exists(), abs_file_path + # print("THIS Is ABS FILEPATH: ", file_path) + # print("THIS IS ABS FILEPATH NAME: ", abs_file_path.name) + # print("THIS Is ABS FILEPATH PARENT: ", abs_file_path.parent) references = _extract_references_from_file( - abs_file_path.parent, Path(abs_file_path.name) + abs_file_path.parent, Path(abs_file_path.name), file_path ) all_need_references.extend(references) store_source_code_links_with_metadata_json( diff --git a/src/extensions/score_source_code_linker/generate_source_code_links_json.py b/src/extensions/score_source_code_linker/generate_source_code_links_json.py index abedc2db4..1c240e0ef 100644 --- a/src/extensions/score_source_code_linker/generate_source_code_links_json.py +++ b/src/extensions/score_source_code_linker/generate_source_code_links_json.py @@ -19,6 +19,8 @@ import os from pathlib import Path +from sphinx_needs.logging import get_logger +LOGGER = get_logger(__name__) from src.extensions.score_source_code_linker.needlinks import ( NeedLink, @@ -43,21 +45,21 @@ def _extract_references_from_line(line: str): yield tag, req.strip() -def _extract_references_from_file(root: Path, file_path: Path) -> list[NeedLink]: +def _extract_references_from_file(root: Path, file_path_name: Path, file_path: Path) -> list[NeedLink]: """Scan a single file for template strings and return findings.""" assert root.is_absolute(), "Root path must be absolute" - assert not file_path.is_absolute(), "File path must be relative to the root" + assert not file_path_name.is_absolute(), "File path must be relative to the root" # assert file_path.is_relative_to(root), ( # f"File path ({file_path}) must be relative to the root ({root})" # ) - assert (root / file_path).exists(), ( - f"File {file_path} does not exist in root {root}." + assert (root / file_path_name).exists(), ( + f"File {file_path_name} does not exist in root {root}." ) findings: list[NeedLink] = [] try: - with open(root / file_path, encoding="utf-8", errors="ignore") as f: + with open(root / file_path_name, encoding="utf-8", errors="ignore") as f: for line_num, line in enumerate(f, 1): for tag, req in _extract_references_from_line(line): findings.append( @@ -69,8 +71,9 @@ def _extract_references_from_file(root: Path, file_path: Path) -> list[NeedLink] full_line=line.strip(), ) ) - except (UnicodeDecodeError, PermissionError, OSError): + except (UnicodeDecodeError, PermissionError, OSError) as e: # Skip files that can't be read as text + LOGGER.debug(f"Error reading file to parse for linked needs: \n{e}") pass return findings @@ -121,8 +124,8 @@ def find_all_need_references(search_path: Path) -> list[NeedLink]: all_need_references.extend(references) elapsed_time = os.times().elapsed - start_time - print( - f"DEBUG: Found {len(all_need_references)} need references " + LOGGER.debug( + f"Found {len(all_need_references)} need references " f"in {elapsed_time:.2f} seconds" ) From c2861cd1c6f70558f1f2d795d9dc91a5b971cb08 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Thu, 12 Mar 2026 13:23:22 +0100 Subject: [PATCH 03/34] Remove metadata file --- scripts_bazel/generate_sourcelinks_cli.py | 2 +- .../score_source_code_linker/metadata.py | 23 ------------------- .../score_source_code_linker/needlinks.py | 14 ++++++++--- 3 files changed, 12 insertions(+), 27 deletions(-) delete mode 100644 src/extensions/score_source_code_linker/metadata.py diff --git a/scripts_bazel/generate_sourcelinks_cli.py b/scripts_bazel/generate_sourcelinks_cli.py index 2f26784ff..6d9336e23 100644 --- a/scripts_bazel/generate_sourcelinks_cli.py +++ b/scripts_bazel/generate_sourcelinks_cli.py @@ -27,8 +27,8 @@ ) from src.extensions.score_source_code_linker.needlinks import ( store_source_code_links_with_metadata_json, + MetaData, ) -from src.extensions.score_source_code_linker.metadata import MetaData logging.basicConfig(level=logging.INFO, format="%(message)s") logger = logging.getLogger(__name__) diff --git a/src/extensions/score_source_code_linker/metadata.py b/src/extensions/score_source_code_linker/metadata.py deleted file mode 100644 index 03acc537a..000000000 --- a/src/extensions/score_source_code_linker/metadata.py +++ /dev/null @@ -1,23 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -from typing import TypedDict, TypeGuard - -class MetaData(TypedDict): - module_name: str - hash: str - url: str - -def is_metadata(x: object) -> TypeGuard[MetaData]: - # Make this as strict/loose as you want; at minimum, it must be a dict. - return isinstance(x, dict) and {"module_name", "hash", "url"} <= x.keys() diff --git a/src/extensions/score_source_code_linker/needlinks.py b/src/extensions/score_source_code_linker/needlinks.py index a727025bd..abcdd0dbb 100644 --- a/src/extensions/score_source_code_linker/needlinks.py +++ b/src/extensions/score_source_code_linker/needlinks.py @@ -16,8 +16,17 @@ import os from dataclasses import asdict, dataclass from pathlib import Path -from typing import Any, TypeGuard -from src.extensions.score_source_code_linker.metadata import MetaData, is_metadata +from typing import Any, TypeGuard, TypedDict + + +class MetaData(TypedDict): + module_name: str + hash: str + url: str + +def is_metadata(x: object) -> TypeGuard[MetaData]: + # Make this as strict/loose as you want; at minimum, it must be a dict. + return isinstance(x, dict) and {"module_name", "hash", "url"} <= x.keys() @dataclass(order=True) @@ -187,4 +196,3 @@ def load_source_code_links_json(file: Path) -> list[NeedLink]: "All items in source_code_links should be NeedLink objects." ) return links - From fb5bdc57e88d96627a64254c6081379aed84a60b Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Thu, 12 Mar 2026 13:24:21 +0100 Subject: [PATCH 04/34] Formatting & Linting --- scripts_bazel/merge_sourcelinks.py | 7 +++---- .../score_source_code_linker/__init__.py | 16 +++++++--------- .../generate_source_code_links_json.py | 2 ++ .../module_source_links.py | 13 ++++++------- .../need_source_links.py | 2 +- src/helper_lib/additional_functions.py | 3 ++- 6 files changed, 21 insertions(+), 22 deletions(-) diff --git a/scripts_bazel/merge_sourcelinks.py b/scripts_bazel/merge_sourcelinks.py index 034d45db5..aa5ece761 100644 --- a/scripts_bazel/merge_sourcelinks.py +++ b/scripts_bazel/merge_sourcelinks.py @@ -20,7 +20,6 @@ import logging import sys from pathlib import Path -from typing import Any # from src.extensions.score_source_code_linker.need_source_links import ( # store_source_code_links_combined_json, @@ -44,8 +43,8 @@ """ -if bazel-out/k8-fastbuild/bin/external/ in file_path => module is external -otherwise it's local +if bazel-out/k8-fastbuild/bin/external/ in file_path => module is external +otherwise it's local if local => module_name & hash == empty if external => parse thing for module_name => look up known_good json for hash & url """ @@ -56,7 +55,7 @@ def parse_info_from_known_good( ) -> tuple[str, str]: print("===THIS IS MODULE NAME WE LOOK FOR===========") print(module_name) - with open(known_good_json, "r") as f: + with open(known_good_json) as f: kg_json = json.load(f) for category in kg_json["modules"].values(): print("===THIS IS CATEGORY=========") diff --git a/src/extensions/score_source_code_linker/__init__.py b/src/extensions/score_source_code_linker/__init__.py index 7e0ff0252..d82994dd0 100644 --- a/src/extensions/score_source_code_linker/__init__.py +++ b/src/extensions/score_source_code_linker/__init__.py @@ -34,20 +34,18 @@ from src.extensions.score_source_code_linker.generate_source_code_links_json import ( generate_source_code_links_json, ) +from src.extensions.score_source_code_linker.module_source_links import ( + group_needs_by_module, + load_module_source_links_json, + store_module_source_links_json, +) from src.extensions.score_source_code_linker.need_source_links import ( NeedSourceLinks, SourceCodeLinks, + group_by_need, load_source_code_links_combined_json, store_source_code_links_combined_json, - group_by_need, ) - -from src.extensions.score_source_code_linker.module_source_links import ( - group_needs_by_module, - store_module_source_links_json, - load_module_source_links_json, -) - from src.extensions.score_source_code_linker.needlinks import ( NeedLink, load_source_code_links_json, @@ -376,7 +374,7 @@ def inject_links_into_needs(app: Sphinx, env: BuildEnvironment) -> None: continue need_as_dict = cast(dict[str, object], need) - metadata = module_grouped_needs.module_name + metadata = module_grouped_needs.module need_as_dict["source_code_link"] = ", ".join( f"{get_github_link(metadata, n)}<>{n.file}:{n.line}" for n in source_code_links.links.CodeLinks diff --git a/src/extensions/score_source_code_linker/generate_source_code_links_json.py b/src/extensions/score_source_code_linker/generate_source_code_links_json.py index 1c240e0ef..ce6839b80 100644 --- a/src/extensions/score_source_code_linker/generate_source_code_links_json.py +++ b/src/extensions/score_source_code_linker/generate_source_code_links_json.py @@ -19,7 +19,9 @@ import os from pathlib import Path + from sphinx_needs.logging import get_logger + LOGGER = get_logger(__name__) from src.extensions.score_source_code_linker.needlinks import ( diff --git a/src/extensions/score_source_code_linker/module_source_links.py b/src/extensions/score_source_code_linker/module_source_links.py index 3957733f0..fe757b947 100644 --- a/src/extensions/score_source_code_linker/module_source_links.py +++ b/src/extensions/score_source_code_linker/module_source_links.py @@ -12,11 +12,10 @@ # ******************************************************************************* -from dataclasses import dataclass, asdict, field import json -from typing import Any +from dataclasses import asdict, dataclass, field from pathlib import Path - +from typing import Any from src.extensions.score_source_code_linker.need_source_links import ( NeedSourceLinks, @@ -36,7 +35,7 @@ class moduleInfo: @dataclass class ModuleSourceLinks: - module_name: moduleInfo + module: moduleInfo needs: list[SourceCodeLinks] = field(default_factory=list) @@ -62,7 +61,7 @@ def ModuleSourceLinks_JSON_Decoder( module_name = d["module_name"] needs = d["needs"] return ModuleSourceLinks( - module_name=moduleInfo( + module=moduleInfo( name=module_name.get("module_name"), hash=module_name.get("hash"), url=module_name.get("url"), @@ -118,13 +117,13 @@ def group_needs_by_module(links: list[SourceCodeLinks]) -> list[ModuleSourceLink if module_key not in module_groups: module_groups[module_key] = ModuleSourceLinks( - module_name=moduleInfo(name=module_key, hash=first_link.hash, url=first_link.url) + module=moduleInfo(name=module_key, hash=first_link.hash, url=first_link.url) ) module_groups[module_key].needs.append(source_link) # Much clearer! return [ - ModuleSourceLinks(module_name=group.module_name, needs=group.needs) + ModuleSourceLinks(module=group.module, needs=group.needs) for group in module_groups.values() ] diff --git a/src/extensions/score_source_code_linker/need_source_links.py b/src/extensions/score_source_code_linker/need_source_links.py index 2deb4c5e8..1823ba4b4 100644 --- a/src/extensions/score_source_code_linker/need_source_links.py +++ b/src/extensions/score_source_code_linker/need_source_links.py @@ -20,8 +20,8 @@ # req-Id: tool_req__docs_dd_link_source_code_link import json -from dataclasses import asdict, dataclass, field from collections import defaultdict +from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any diff --git a/src/helper_lib/additional_functions.py b/src/helper_lib/additional_functions.py index 872d66af2..049a14548 100644 --- a/src/helper_lib/additional_functions.py +++ b/src/helper_lib/additional_functions.py @@ -12,9 +12,10 @@ # ******************************************************************************* from pathlib import Path +from src.extensions.score_source_code_linker.module_source_links import moduleInfo + # Import types that depend on score_source_code_linker from src.extensions.score_source_code_linker.needlinks import DefaultNeedLink, NeedLink -from src.extensions.score_source_code_linker.module_source_links import moduleInfo from src.extensions.score_source_code_linker.testlink import ( DataForTestLink, DataOfTestCase, From 8ea74178ce76d900d0ee72978737ade17fb8b5b1 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Thu, 12 Mar 2026 13:26:11 +0100 Subject: [PATCH 05/34] Known_good required in merge script --- scripts_bazel/merge_sourcelinks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts_bazel/merge_sourcelinks.py b/scripts_bazel/merge_sourcelinks.py index aa5ece761..43785ea91 100644 --- a/scripts_bazel/merge_sourcelinks.py +++ b/scripts_bazel/merge_sourcelinks.py @@ -80,7 +80,7 @@ def main(): ) _ = parser.add_argument( "--known_good", - default=None, + required=True, help="Optional path to a 'known good' JSON file (provided by Bazel).", ) _ = parser.add_argument( From f818cf26d03a359d1330db68c48c12fc5a22c5b4 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Thu, 12 Mar 2026 13:30:37 +0100 Subject: [PATCH 06/34] Remove debug print statements --- scripts_bazel/generate_sourcelinks_cli.py | 17 +++-------------- scripts_bazel/merge_sourcelinks.py | 8 +------- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/scripts_bazel/generate_sourcelinks_cli.py b/scripts_bazel/generate_sourcelinks_cli.py index 6d9336e23..8b1c4bfd5 100644 --- a/scripts_bazel/generate_sourcelinks_cli.py +++ b/scripts_bazel/generate_sourcelinks_cli.py @@ -44,15 +44,11 @@ def parse_module_name_from_path(path: Path) -> str: # COMBO BUILD # If external is in the filepath that gets parsed => # file is in an external module => combo build - # e.g. .../external/score_docs_as_code+/src/helper_lib/__init__.py + # Example Path: + # PosixPath('external/score_docs_as_code+/src/helper_lib/test_helper_lib.py' - # PATH if we are in local repository - # PosixPath('src/helper_lib/test_helper_lib.py') - # Path if we are in combo build and externally - # PosixPath('external/score_docs_as_code+/src/helper_lib/test_helper_lib.py' - print("======== THIs IS PATH we PARSIGN FOr MODULE NAME") - print(path) if str(path).startswith("external/"): + # This allows for files / folders etc. to have `external` in their name too. module_raw = str(path).removeprefix("external/") filepath_split = str(module_raw).split("/", maxsplit=1) module_name = str(filepath_split[0].removesuffix("+")) @@ -89,16 +85,9 @@ def main(): for file_path in args.files: if "known_good.json" not in str(file_path) and not metadata_set: metadata["module_name"] = parse_module_name_from_path(file_path) - # print("================") - # print(metadata) - # print("===============") - # print("METADATA SET") metadata_set = True abs_file_path = file_path.resolve() assert abs_file_path.exists(), abs_file_path - # print("THIS Is ABS FILEPATH: ", file_path) - # print("THIS IS ABS FILEPATH NAME: ", abs_file_path.name) - # print("THIS Is ABS FILEPATH PARENT: ", abs_file_path.parent) references = _extract_references_from_file( abs_file_path.parent, Path(abs_file_path.name), file_path ) diff --git a/scripts_bazel/merge_sourcelinks.py b/scripts_bazel/merge_sourcelinks.py index 43785ea91..b956b0bda 100644 --- a/scripts_bazel/merge_sourcelinks.py +++ b/scripts_bazel/merge_sourcelinks.py @@ -53,16 +53,10 @@ def parse_info_from_known_good( known_good_json: Path, module_name: str ) -> tuple[str, str]: - print("===THIS IS MODULE NAME WE LOOK FOR===========") - print(module_name) with open(known_good_json) as f: kg_json = json.load(f) for category in kg_json["modules"].values(): - print("===THIS IS CATEGORY=========") - print(category) if module_name in category: - print("===THIS IS MODULE NAME INSIDe CATEGORY===========") - print(module_name) m = category[module_name] return (m["hash"], m["repo"].removesuffix(".git")) raise KeyError(f"Module {module_name!r} not found in known_good_json.") @@ -104,7 +98,7 @@ def main(): ) metadata["hash"] = hash metadata["url"] = repo - # In the case that 'metadata[module_name]' is empty + # In the case that 'metadata[module_name]' is 'local_module' or empty (somehow) # hash & url are already existing and empty inside of 'metadata' # Therefore all 3 keys will be written to needlinks in each branch for d in data[1:]: From adeae4f8501a3c31369528fba7e3d8d4019c24d4 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Thu, 12 Mar 2026 13:40:57 +0100 Subject: [PATCH 07/34] Copilot Findings --- .../score_source_code_linker/__init__.py | 4 ++-- .../module_source_links.py | 14 ++++++++------ .../score_source_code_linker/needlinks.py | 9 +++------ src/helper_lib/additional_functions.py | 6 +++--- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/extensions/score_source_code_linker/__init__.py b/src/extensions/score_source_code_linker/__init__.py index d82994dd0..6a409b917 100644 --- a/src/extensions/score_source_code_linker/__init__.py +++ b/src/extensions/score_source_code_linker/__init__.py @@ -226,9 +226,9 @@ def register_combined_linker(app: Sphinx): def setup_combined_linker(app: Sphinx, _: BuildEnvironment): grouped_cache = get_cache_filename(app.outdir, "score_scl_grouped_cache.json") - gruped_cache_exists = grouped_cache.exists() + grouped_cache_exists = grouped_cache.exists() # TODO this cache should be done via Bazel - if not gruped_cache_exists or not app.config.skip_rescanning_via_source_code_linker: + if not grouped_cache_exists or not app.config.skip_rescanning_via_source_code_linker: LOGGER.debug( "Did not find combined json 'score_scl_grouped_cache.json' in _build." "Generating new one" diff --git a/src/extensions/score_source_code_linker/module_source_links.py b/src/extensions/score_source_code_linker/module_source_links.py index fe757b947..9197b75ff 100644 --- a/src/extensions/score_source_code_linker/module_source_links.py +++ b/src/extensions/score_source_code_linker/module_source_links.py @@ -27,7 +27,7 @@ @dataclass -class moduleInfo: +class ModuleInfo: name: str hash: str url: str @@ -35,7 +35,7 @@ class moduleInfo: @dataclass class ModuleSourceLinks: - module: moduleInfo + module: ModuleInfo needs: list[SourceCodeLinks] = field(default_factory=list) @@ -48,7 +48,7 @@ def default(self, o: object): if isinstance(o, NeedLink): return o.to_dict_without_metadata() if isinstance( - o, ModuleSourceLinks | SourceCodeLinks | DataForTestLink | NeedSourceLinks + o, (ModuleSourceLinks, SourceCodeLinks, DataForTestLink, NeedSourceLinks) ): return asdict(o) return super().default(o) @@ -61,14 +61,14 @@ def ModuleSourceLinks_JSON_Decoder( module_name = d["module_name"] needs = d["needs"] return ModuleSourceLinks( - module=moduleInfo( + module=ModuleInfo( name=module_name.get("module_name"), hash=module_name.get("hash"), url=module_name.get("url"), ), # We know this can only be list[SourceCodeLinks] and nothing else # Therefore => we ignore the type error here - needs=[SourceCodeLinks_JSON_Decoder(need) for need in needs] # type: ignore + needs=[SourceCodeLinks_JSON_Decoder(need) for need in needs], # type: ignore ) return d @@ -117,7 +117,9 @@ def group_needs_by_module(links: list[SourceCodeLinks]) -> list[ModuleSourceLink if module_key not in module_groups: module_groups[module_key] = ModuleSourceLinks( - module=moduleInfo(name=module_key, hash=first_link.hash, url=first_link.url) + module=ModuleInfo( + name=module_key, hash=first_link.hash, url=first_link.url + ) ) module_groups[module_key].needs.append(source_link) # Much clearer! diff --git a/src/extensions/score_source_code_linker/needlinks.py b/src/extensions/score_source_code_linker/needlinks.py index abcdd0dbb..324e54114 100644 --- a/src/extensions/score_source_code_linker/needlinks.py +++ b/src/extensions/score_source_code_linker/needlinks.py @@ -42,10 +42,10 @@ class NeedLink: hash: str = "" url: str = "" - def to_dict_full(self) -> dict[str, str | Path]: + def to_dict_full(self) -> dict[str, str | Path | int]: return asdict(self) - def to_dict_without_metadata(self) -> dict[str, str | Path]: + def to_dict_without_metadata(self) -> dict[str, str | Path | int]: d = asdict(self) d.pop("module_name", None) d.pop("hash", None) @@ -114,10 +114,7 @@ def store_source_code_links_with_metadata_json( def store_source_code_links_json(file: Path, needlist: list[NeedLink]) -> None: """ Writes a JSON array: - [ meta_dict, needlink1, needlink2, ... ] - - meta_dict must include: - module_name, hash, url + [ needlink1, needlink2, ... ] """ file.parent.mkdir(exist_ok=True) diff --git a/src/helper_lib/additional_functions.py b/src/helper_lib/additional_functions.py index 049a14548..21b4d6fd2 100644 --- a/src/helper_lib/additional_functions.py +++ b/src/helper_lib/additional_functions.py @@ -12,7 +12,7 @@ # ******************************************************************************* from pathlib import Path -from src.extensions.score_source_code_linker.module_source_links import moduleInfo +from src.extensions.score_source_code_linker.module_source_links import ModuleInfo # Import types that depend on score_source_code_linker from src.extensions.score_source_code_linker.needlinks import DefaultNeedLink, NeedLink @@ -28,7 +28,7 @@ def get_github_link( - metadata: moduleInfo, + metadata: ModuleInfo, link: NeedLink | DataForTestLink | DataOfTestCase | None = None, ) -> str: if link is None: @@ -54,7 +54,7 @@ def get_github_link_from_git( def get_github_link_from_json( - metadata: moduleInfo, + metadata: ModuleInfo, link: NeedLink | DataForTestLink | DataOfTestCase | None = None, ): if link is None: From b92bf6e912a24081946b2427c786d01e2ecbfb95 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Thu, 12 Mar 2026 14:06:22 +0100 Subject: [PATCH 08/34] Formatting & Linting --- scripts_bazel/generate_sourcelinks_cli.py | 5 ++--- scripts_bazel/merge_sourcelinks.py | 14 +------------- .../score_source_code_linker/__init__.py | 12 ++++-------- .../generate_source_code_links_json.py | 8 +++++--- .../module_source_links.py | 2 +- .../score_source_code_linker/needlinks.py | 2 +- 6 files changed, 14 insertions(+), 29 deletions(-) diff --git a/scripts_bazel/generate_sourcelinks_cli.py b/scripts_bazel/generate_sourcelinks_cli.py index 8b1c4bfd5..ba54b5d23 100644 --- a/scripts_bazel/generate_sourcelinks_cli.py +++ b/scripts_bazel/generate_sourcelinks_cli.py @@ -26,8 +26,8 @@ _extract_references_from_file, # pyright: ignore[reportPrivateUsage] TODO: move it out of the extension and into this script ) from src.extensions.score_source_code_linker.needlinks import ( - store_source_code_links_with_metadata_json, MetaData, + store_source_code_links_with_metadata_json, ) logging.basicConfig(level=logging.INFO, format="%(message)s") @@ -51,8 +51,7 @@ def parse_module_name_from_path(path: Path) -> str: # This allows for files / folders etc. to have `external` in their name too. module_raw = str(path).removeprefix("external/") filepath_split = str(module_raw).split("/", maxsplit=1) - module_name = str(filepath_split[0].removesuffix("+")) - return module_name + return str(filepath_split[0].removesuffix("+")) return "local_module" diff --git a/scripts_bazel/merge_sourcelinks.py b/scripts_bazel/merge_sourcelinks.py index b956b0bda..626b48c56 100644 --- a/scripts_bazel/merge_sourcelinks.py +++ b/scripts_bazel/merge_sourcelinks.py @@ -28,18 +28,6 @@ logging.basicConfig(level=logging.INFO, format="%(message)s") logger = logging.getLogger(__name__) -# [ -# PosixPath('bazel-out/k8-fastbuild/bin/sourcelinks_json.json'), -# PosixPath('bazel-out/k8-fastbuild/bin/external/score_persistency+/sourcelinks_json.json'), -# PosixPath('bazel-out/k8-fastbuild/bin/external/score_orchestrator+/sourcelinks_json.json'), -# PosixPath('bazel-out/k8-fastbuild/bin/external/score_kyron+/sourcelinks_json.json'), -# PosixPath('bazel-out/k8-fastbuild/bin/external/score_baselibs+/sourcelinks_json.json'), -# PosixPath('bazel-out/k8-fastbuild/bin/external/score_baselibs_rust+/sourcelinks_json.json'), -# PosixPath('bazel-out/k8-fastbuild/bin/external/score_logging+/sourcelinks_json.json'), -# PosixPath('bazel-out/k8-fastbuild/bin/external/score_platform+/sourcelinks_json.json'), -# PosixPath('bazel-out/k8-fastbuild/bin/external/score_process+/sourcelinks_json.json'), -# PosixPath('bazel-out/k8-fastbuild/bin/external/score_docs_as_code+/sourcelinks_json.json') -# ] """ @@ -98,7 +86,7 @@ def main(): ) metadata["hash"] = hash metadata["url"] = repo - # In the case that 'metadata[module_name]' is 'local_module' or empty (somehow) + # In the case that 'metadata[module_name]' is 'local_module' # hash & url are already existing and empty inside of 'metadata' # Therefore all 3 keys will be written to needlinks in each branch for d in data[1:]: diff --git a/src/extensions/score_source_code_linker/__init__.py b/src/extensions/score_source_code_linker/__init__.py index 6a409b917..a84ddfb42 100644 --- a/src/extensions/score_source_code_linker/__init__.py +++ b/src/extensions/score_source_code_linker/__init__.py @@ -40,19 +40,15 @@ store_module_source_links_json, ) from src.extensions.score_source_code_linker.need_source_links import ( - NeedSourceLinks, - SourceCodeLinks, group_by_need, load_source_code_links_combined_json, store_source_code_links_combined_json, ) from src.extensions.score_source_code_linker.needlinks import ( - NeedLink, load_source_code_links_json, load_source_code_links_with_metadata_json, ) from src.extensions.score_source_code_linker.testlink import ( - DataForTestLink, load_data_of_test_case_json, load_test_xml_parsed_json, ) @@ -228,7 +224,10 @@ def setup_combined_linker(app: Sphinx, _: BuildEnvironment): grouped_cache = get_cache_filename(app.outdir, "score_scl_grouped_cache.json") grouped_cache_exists = grouped_cache.exists() # TODO this cache should be done via Bazel - if not grouped_cache_exists or not app.config.skip_rescanning_via_source_code_linker: + if ( + not grouped_cache_exists + or not app.config.skip_rescanning_via_source_code_linker + ): LOGGER.debug( "Did not find combined json 'score_scl_grouped_cache.json' in _build." "Generating new one" @@ -351,9 +350,6 @@ def inject_links_into_needs(app: Sphinx, env: BuildEnvironment) -> None: scl_by_module = load_module_source_links_json( get_cache_filename(app.outdir, "score_module_grouped_scl_cache.json") ) - # source_code_links_by_need = load_source_code_links_combined_json( - # get_cache_filename(app.outdir, "score_scl_grouped_cache.json") - # ) for module_grouped_needs in scl_by_module: for source_code_links in module_grouped_needs.needs: need = find_need(needs_copy, source_code_links.need) diff --git a/src/extensions/score_source_code_linker/generate_source_code_links_json.py b/src/extensions/score_source_code_linker/generate_source_code_links_json.py index ce6839b80..33ef6af37 100644 --- a/src/extensions/score_source_code_linker/generate_source_code_links_json.py +++ b/src/extensions/score_source_code_linker/generate_source_code_links_json.py @@ -22,13 +22,13 @@ from sphinx_needs.logging import get_logger -LOGGER = get_logger(__name__) - from src.extensions.score_source_code_linker.needlinks import ( NeedLink, store_source_code_links_json, ) +LOGGER = get_logger(__name__) + TAGS = [ "# " + "req-traceability:", "# " + "req-Id:", @@ -47,7 +47,9 @@ def _extract_references_from_line(line: str): yield tag, req.strip() -def _extract_references_from_file(root: Path, file_path_name: Path, file_path: Path) -> list[NeedLink]: +def _extract_references_from_file( + root: Path, file_path_name: Path, file_path: Path +) -> list[NeedLink]: """Scan a single file for template strings and return findings.""" assert root.is_absolute(), "Root path must be absolute" assert not file_path_name.is_absolute(), "File path must be relative to the root" diff --git a/src/extensions/score_source_code_linker/module_source_links.py b/src/extensions/score_source_code_linker/module_source_links.py index 9197b75ff..887d4b667 100644 --- a/src/extensions/score_source_code_linker/module_source_links.py +++ b/src/extensions/score_source_code_linker/module_source_links.py @@ -48,7 +48,7 @@ def default(self, o: object): if isinstance(o, NeedLink): return o.to_dict_without_metadata() if isinstance( - o, (ModuleSourceLinks, SourceCodeLinks, DataForTestLink, NeedSourceLinks) + o, ModuleSourceLinks | SourceCodeLinks | DataForTestLink | NeedSourceLinks ): return asdict(o) return super().default(o) diff --git a/src/extensions/score_source_code_linker/needlinks.py b/src/extensions/score_source_code_linker/needlinks.py index 324e54114..8bc8f9439 100644 --- a/src/extensions/score_source_code_linker/needlinks.py +++ b/src/extensions/score_source_code_linker/needlinks.py @@ -16,7 +16,7 @@ import os from dataclasses import asdict, dataclass from pathlib import Path -from typing import Any, TypeGuard, TypedDict +from typing import Any, TypedDict, TypeGuard class MetaData(TypedDict): From 5a08e5da679545e32c31386010708498da6b3633 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Thu, 12 Mar 2026 14:11:28 +0100 Subject: [PATCH 09/34] Basepyright linting --- .../score_source_code_linker/generate_source_code_links_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extensions/score_source_code_linker/generate_source_code_links_json.py b/src/extensions/score_source_code_linker/generate_source_code_links_json.py index 33ef6af37..7b39b7228 100644 --- a/src/extensions/score_source_code_linker/generate_source_code_links_json.py +++ b/src/extensions/score_source_code_linker/generate_source_code_links_json.py @@ -124,7 +124,7 @@ def find_all_need_references(search_path: Path) -> list[NeedLink]: # Use os.walk to have better control over directory traversal for file in iterate_files_recursively(search_path): - references = _extract_references_from_file(search_path, file) + references = _extract_references_from_file(search_path,Path(file.name), file) all_need_references.extend(references) elapsed_time = os.times().elapsed - start_time From ce313871ed1e07d268fe88546ee01a37f07fb448 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Fri, 13 Mar 2026 16:51:28 +0100 Subject: [PATCH 10/34] WIP: Testlinks endlink still wrong --- scripts_bazel/BUILD | 1 + scripts_bazel/generate_sourcelinks_cli.py | 23 +------- scripts_bazel/merge_sourcelinks.py | 22 +++----- src/extensions/score_source_code_linker/BUILD | 1 + .../score_source_code_linker/__init__.py | 2 +- .../score_source_code_linker/helpers.py} | 36 ++++++++++++ .../module_source_links.py | 42 ++++++++------ .../score_source_code_linker/testlink.py | 34 +++++++++++ .../tests/test_codelink.py | 2 +- .../test_source_code_link_integration.py | 3 +- .../score_source_code_linker/xml_parser.py | 56 ++++++++++++++++--- src/helper_lib/BUILD | 3 +- src/helper_lib/__init__.py | 2 +- 13 files changed, 160 insertions(+), 67 deletions(-) rename src/{helper_lib/additional_functions.py => extensions/score_source_code_linker/helpers.py} (60%) diff --git a/scripts_bazel/BUILD b/scripts_bazel/BUILD index 81c9212f1..befe51730 100644 --- a/scripts_bazel/BUILD +++ b/scripts_bazel/BUILD @@ -33,6 +33,7 @@ py_binary( py_binary( name = "merge_sourcelinks", srcs = ["merge_sourcelinks.py"], + deps= [ "//src/extensions/score_source_code_linker"], main = "merge_sourcelinks.py", visibility = ["//visibility:public"], ) diff --git a/scripts_bazel/generate_sourcelinks_cli.py b/scripts_bazel/generate_sourcelinks_cli.py index ba54b5d23..e1feb8e0b 100644 --- a/scripts_bazel/generate_sourcelinks_cli.py +++ b/scripts_bazel/generate_sourcelinks_cli.py @@ -25,36 +25,17 @@ from src.extensions.score_source_code_linker.generate_source_code_links_json import ( _extract_references_from_file, # pyright: ignore[reportPrivateUsage] TODO: move it out of the extension and into this script ) +from src.extensions.score_source_code_linker.helpers import parse_module_name_from_path from src.extensions.score_source_code_linker.needlinks import ( MetaData, store_source_code_links_with_metadata_json, ) + logging.basicConfig(level=logging.INFO, format="%(message)s") logger = logging.getLogger(__name__) -def parse_module_name_from_path(path: Path) -> str: - """ - Parse out the Module-Name from the filename gotten - /home/user/.cache/bazel/aksj37981712/external/score_docs_as_code+/src/tests/testfile.py - => score_docs_as_code - """ - - # COMBO BUILD - # If external is in the filepath that gets parsed => - # file is in an external module => combo build - # Example Path: - # PosixPath('external/score_docs_as_code+/src/helper_lib/test_helper_lib.py' - - if str(path).startswith("external/"): - # This allows for files / folders etc. to have `external` in their name too. - module_raw = str(path).removeprefix("external/") - filepath_split = str(module_raw).split("/", maxsplit=1) - return str(filepath_split[0].removesuffix("+")) - return "local_module" - - def main(): parser = argparse.ArgumentParser( description="Generate source code links JSON from source files" diff --git a/scripts_bazel/merge_sourcelinks.py b/scripts_bazel/merge_sourcelinks.py index 626b48c56..afb2cc198 100644 --- a/scripts_bazel/merge_sourcelinks.py +++ b/scripts_bazel/merge_sourcelinks.py @@ -21,9 +21,7 @@ import sys from pathlib import Path -# from src.extensions.score_source_code_linker.need_source_links import ( -# store_source_code_links_combined_json, -# ) +from src.extensions.score_source_code_linker.helpers import parse_info_from_known_good logging.basicConfig(level=logging.INFO, format="%(message)s") logger = logging.getLogger(__name__) @@ -38,17 +36,10 @@ """ -def parse_info_from_known_good( - known_good_json: Path, module_name: str -) -> tuple[str, str]: - with open(known_good_json) as f: - kg_json = json.load(f) - for category in kg_json["modules"].values(): - if module_name in category: - m = category[module_name] - return (m["hash"], m["repo"].removesuffix(".git")) - raise KeyError(f"Module {module_name!r} not found in known_good_json.") +def add_needid_to_metaneed_mapping(mapping: dict[str, dict[str, str]], metadata: dict[str, str], needid: str): + mapping + pass def main(): parser = argparse.ArgumentParser( @@ -76,6 +67,7 @@ def main(): all_files = [x for x in args.files if "known_good.json" not in str(x)] merged = [] + needs_metadata_mapping = {} for json_file in all_files: with open(json_file) as f: data = json.load(f) @@ -89,11 +81,11 @@ def main(): # In the case that 'metadata[module_name]' is 'local_module' # hash & url are already existing and empty inside of 'metadata' # Therefore all 3 keys will be written to needlinks in each branch + for d in data[1:]: d.update(metadata) assert isinstance(data, list), repr(data) - merged.extend(data[1:]) - + merged.extend(data[1:]) with open(args.output, "w") as f: json.dump(merged, f, indent=2, ensure_ascii=False) diff --git a/src/extensions/score_source_code_linker/BUILD b/src/extensions/score_source_code_linker/BUILD index e3c289c66..55d471374 100644 --- a/src/extensions/score_source_code_linker/BUILD +++ b/src/extensions/score_source_code_linker/BUILD @@ -54,6 +54,7 @@ py_library( "needlinks.py", "testlink.py", "xml_parser.py", + "helpers.py", ], imports = ["."], visibility = ["//visibility:public"], diff --git a/src/extensions/score_source_code_linker/__init__.py b/src/extensions/score_source_code_linker/__init__.py index a84ddfb42..c687e2e3e 100644 --- a/src/extensions/score_source_code_linker/__init__.py +++ b/src/extensions/score_source_code_linker/__init__.py @@ -60,7 +60,7 @@ find_git_root, find_ws_root, ) -from src.helper_lib.additional_functions import get_github_link +from src.extensions.score_source_code_linker.helpers import get_github_link LOGGER = get_logger(__name__) # Uncomment this to enable more verbose logging diff --git a/src/helper_lib/additional_functions.py b/src/extensions/score_source_code_linker/helpers.py similarity index 60% rename from src/helper_lib/additional_functions.py rename to src/extensions/score_source_code_linker/helpers.py index 21b4d6fd2..5544e2696 100644 --- a/src/helper_lib/additional_functions.py +++ b/src/extensions/score_source_code_linker/helpers.py @@ -10,6 +10,7 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* +import json from pathlib import Path from src.extensions.score_source_code_linker.module_source_links import ModuleInfo @@ -62,3 +63,38 @@ def get_github_link_from_json( base_url = metadata.url current_hash = metadata.hash return f"{base_url}/blob/{current_hash}/{link.file}#L{link.line}" + + +def parse_module_name_from_path(path: Path) -> str: + """ + Parse out the Module-Name from the filename gotten + /home/user/.cache/bazel/aksj37981712/external/score_docs_as_code+/src/tests/testfile.py + => score_docs_as_code + """ + + # COMBO BUILD + # If external is in the filepath that gets parsed => + # file is in an external module => combo build + # Example Path: + # PosixPath('external/score_docs_as_code+/src/helper_lib/test_helper_lib.py' + + if str(path).startswith("external/"): + # This allows for files / folders etc. to have `external` in their name too. + module_raw = str(path).removeprefix("external/") + filepath_split = str(module_raw).split("/", maxsplit=1) + return str(filepath_split[0].removesuffix("+")) + # We return this when we are in a local build `//:docs` the rest of DaC knows + # What to do then if it encounters this module_name + return "local_module" + + +def parse_info_from_known_good( + known_good_json: Path, module_name: str +) -> tuple[str, str]: + with open(known_good_json) as f: + kg_json = json.load(f) + for category in kg_json["modules"].values(): + if module_name in category: + m = category[module_name] + return (m["hash"], m["repo"].removesuffix(".git")) + raise KeyError(f"Module {module_name!r} not found in known_good_json.") diff --git a/src/extensions/score_source_code_linker/module_source_links.py b/src/extensions/score_source_code_linker/module_source_links.py index 887d4b667..97dc27a68 100644 --- a/src/extensions/score_source_code_linker/module_source_links.py +++ b/src/extensions/score_source_code_linker/module_source_links.py @@ -15,6 +15,7 @@ import json from dataclasses import asdict, dataclass, field from pathlib import Path +from re import M from typing import Any from src.extensions.score_source_code_linker.need_source_links import ( @@ -43,13 +44,12 @@ class ModuleSourceLinks_JSON_Encoder(json.JSONEncoder): def default(self, o: object): if isinstance(o, Path): return str(o) - # We do not want to save the metadata inside the codelink + # We do not want to save the metadata inside the codelink or testlink + # As we save this already in a structure above it # (hash, module_name, url) - if isinstance(o, NeedLink): + if isinstance(o, NeedLink | DataForTestLink): return o.to_dict_without_metadata() - if isinstance( - o, ModuleSourceLinks | SourceCodeLinks | DataForTestLink | NeedSourceLinks - ): + if isinstance(o, ModuleSourceLinks | SourceCodeLinks | NeedSourceLinks): return asdict(o) return super().default(o) @@ -57,14 +57,14 @@ def default(self, o: object): def ModuleSourceLinks_JSON_Decoder( d: dict[str, Any], ) -> ModuleSourceLinks | dict[str, Any]: - if "module_name" in d and "needs" in d: - module_name = d["module_name"] + if "module" in d and "needs" in d: + module = d["module"] needs = d["needs"] return ModuleSourceLinks( module=ModuleInfo( - name=module_name.get("module_name"), - hash=module_name.get("hash"), - url=module_name.get("url"), + name=module.get("module_name"), + hash=module.get("hash"), + url=module.get("url"), ), # We know this can only be list[SourceCodeLinks] and nothing else # Therefore => we ignore the type error here @@ -95,12 +95,16 @@ def load_module_source_links_json(file: Path) -> list[ModuleSourceLinks]: object_hook=ModuleSourceLinks_JSON_Decoder, ) assert isinstance(links, list), ( - "The combined source code linker links should be " - "a list of SourceCodeLinks objects." + "The ModuleSourceLink json should be aa list of ModuleSourceLink objects." ) + print("=====================") + print("=== TESTING LINKS IN ModuleSourceLink === ") + for link in links: + if not isinstance(link, ModuleSourceLinks): + print(f"Link not module_sourcelink: {link}") + print("=====================") assert all(isinstance(link, ModuleSourceLinks) for link in links), ( - "All items in combined_source_code_linker_cache should be " - "SourceCodeLinks objects." + "All items in module source link cache should be ModuleSourceLink objects." ) return links @@ -109,10 +113,14 @@ def group_needs_by_module(links: list[SourceCodeLinks]) -> list[ModuleSourceLink module_groups: dict[str, ModuleSourceLinks] = {} for source_link in links: - if not source_link.links.CodeLinks: + # Check if we can take moduleInfo from code or testlinks + if source_link.links.CodeLinks: + first_link = source_link.links.CodeLinks[0] + elif source_link.links.TestLinks: + first_link = source_link.links.TestLinks[0] + else: + # This should not happen? continue - - first_link = source_link.links.CodeLinks[0] module_key = first_link.module_name if module_key not in module_groups: diff --git a/src/extensions/score_source_code_linker/testlink.py b/src/extensions/score_source_code_linker/testlink.py index ee83c7f95..f001cc203 100644 --- a/src/extensions/score_source_code_linker/testlink.py +++ b/src/extensions/score_source_code_linker/testlink.py @@ -30,6 +30,7 @@ from sphinx_needs import logging + LOGGER = logging.get_logger(__name__) @@ -42,6 +43,19 @@ class DataForTestLink: verify_type: str result: str result_text: str = "" + module_name: str = "" + hash: str = "" + url: str = "" + + def to_dict_full(self) -> dict[str, str | Path | int]: + return asdict(self) + + def to_dict_without_metadata(self) -> dict[str, str | Path | int]: + d = asdict(self) + d.pop("module_name", None) + d.pop("hash", None) + d.pop("url", None) + return d class DataForTestLink_JSON_Encoder(json.JSONEncoder): @@ -60,6 +74,9 @@ def DataForTestLink_JSON_Decoder(d: dict[str, Any]) -> DataForTestLink | dict[st "line", "need", "verify_type", + "module_name", + "hash", + "url", "result", "result_text", } <= d.keys(): @@ -68,6 +85,9 @@ def DataForTestLink_JSON_Decoder(d: dict[str, Any]) -> DataForTestLink | dict[st file=Path(d["file"]), line=d["line"], need=d["need"], + module_name=d.get("module_name", ""), + hash=d.get("hash", ""), + url=d.get("url", ""), verify_type=d["verify_type"], result=d["result"], result_text=d["result_text"], @@ -83,6 +103,9 @@ class DataOfTestCase: file: str | None = None line: str | None = None result: str | None = None # passed | falied | skipped | disabled + module_name: str | None = None + hash: str | None = None + url: str | None = None # Intentionally not snakecase to make dict parsing simple TestType: str | None = None DerivationTechnique: str | None = None @@ -98,6 +121,9 @@ def from_dict(cls, data: dict[str, Any]): # type-ignore file=data.get("file"), line=data.get("line"), result=data.get("result"), + module_name=data.get("module_name"), + hash=data.get("hash"), + url=data.get("url"), TestType=data.get("TestType"), DerivationTechnique=data.get("DerivationTechnique"), result_text=data.get("result_text"), @@ -158,6 +184,8 @@ def is_valid(self) -> bool: # and self.TestType is not None # and self.DerivationTechnique is not None # ): + # Hash & URL are explictily allowed to be empty but not none. + # module_name has to be always filled or something went wrong fields = [ x for x in self.__dataclass_fields__ @@ -199,6 +227,9 @@ def parse_attributes(verify_field: str | None, verify_type: str): assert self.file is not None assert self.line is not None assert self.result is not None + assert self.module_name is not None + assert self.hash is not None + assert self.url is not None assert self.result_text is not None assert self.TestType is not None assert self.DerivationTechnique is not None @@ -212,6 +243,9 @@ def parse_attributes(verify_field: str | None, verify_type: str): verify_type=verify_type, result=self.result, result_text=self.result_text, + module_name=self.module_name, + hash=self.hash, + url=self.url, ) return list( diff --git a/src/extensions/score_source_code_linker/tests/test_codelink.py b/src/extensions/score_source_code_linker/tests/test_codelink.py index 29ddc7235..9615c0058 100644 --- a/src/extensions/score_source_code_linker/tests/test_codelink.py +++ b/src/extensions/score_source_code_linker/tests/test_codelink.py @@ -43,7 +43,7 @@ from src.helper_lib import ( get_current_git_hash, ) -from src.helper_lib.additional_functions import get_github_link +from src.extensions.score_source_code_linker.helpers import get_github_link """ # ────────────────ATTENTION─────────────── diff --git a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py index 60bb98f80..01bb7ff58 100644 --- a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py +++ b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py @@ -37,7 +37,8 @@ SourceCodeLinks_TEST_JSON_Decoder, ) from src.helper_lib import find_ws_root, get_github_base_url -from src.helper_lib.additional_functions import get_github_link + +from src.extensions.score_source_code_linker.helpers import get_github_link @pytest.fixture() diff --git a/src/extensions/score_source_code_linker/xml_parser.py b/src/extensions/score_source_code_linker/xml_parser.py index 8432e1fc3..6facb983b 100644 --- a/src/extensions/score_source_code_linker/xml_parser.py +++ b/src/extensions/score_source_code_linker/xml_parser.py @@ -33,18 +33,41 @@ from sphinx_needs import logging from sphinx_needs.api import add_external_need +from src.extensions.score_source_code_linker.needlinks import ( + MetaData, +) +from src.extensions.score_source_code_linker.module_source_links import ModuleInfo from src.extensions.score_source_code_linker.testlink import ( DataOfTestCase, store_data_of_test_case_json, store_test_xml_parsed_json, ) from src.helper_lib import find_ws_root -from src.helper_lib.additional_functions import get_github_link +from src.extensions.score_source_code_linker.helpers import ( + get_github_link, + parse_info_from_known_good, + parse_module_name_from_path, +) logger = logging.get_logger(__name__) logger.setLevel("DEBUG") +def get_metadata_from_test_path(filepath: Path) -> MetaData: + known_good_json = os.environ.get("KNOWN_GOOD_JSON") + module_name = parse_module_name_from_path(filepath) + md: MetaData = { + "module_name": module_name, + "hash": "", + "url": "", + } + if module_name != "local_module" and known_good_json: + md["hash"], md["url"] = parse_info_from_known_good( + Path(known_good_json), module_name + ) + return md + + def parse_testcase_result(testcase: ET.Element) -> tuple[str, str]: """ Returns 'result' and 'result_text' found in the 'message' @@ -101,7 +124,7 @@ def read_test_xml_file(file: Path) -> tuple[list[DataOfTestCase], list[str], lis missing_prop_tests: list[str] = [] tree = ET.parse(file) root = tree.getroot() - + md = get_metadata_from_test_path(file) for testsuite in root.findall("testsuite"): for testcase in testsuite.findall("testcase"): case_properties = {} @@ -161,6 +184,7 @@ def read_test_xml_file(file: Path) -> tuple[list[DataOfTestCase], list[str], lis # If the is_valid method would return 'False' anyway. # I just can't think of it right now, leaving this for future me case_properties = parse_properties(case_properties, properties_element) + case_properties.update(md) test_case = DataOfTestCase.from_dict(case_properties) if not test_case.is_valid(): missing_prop_tests.append(testname) @@ -169,6 +193,7 @@ def read_test_xml_file(file: Path) -> tuple[list[DataOfTestCase], list[str], lis return test_case_needs, non_prop_tests, missing_prop_tests +# /home/maximilianp/score_personal/reference_integration/bazel-testlogs/external/score_docs_as_code+/src/helper_lib/helper_lib_tests/test.xml def find_xml_files(dir: Path) -> list[Path]: """ Recursively search all test.xml files inside 'bazel-testlogs' @@ -183,18 +208,21 @@ def find_xml_files(dir: Path) -> list[Path]: for root, _, files in os.walk(dir): if test_file_name in files: xml_paths.append(Path(os.path.join(root, test_file_name))) + print("=========================================") + print(xml_paths[0]) + print("=========================================") return xml_paths -def find_test_folder(base_path: Path | None = None) -> Path | None: +def find_test_folder(base_path: Path | None = None) -> tuple[Path | None, Path | None]: ws_root = base_path if base_path is not None else find_ws_root() assert ws_root is not None if os.path.isdir(ws_root / "tests-report"): - return ws_root / "tests-report" + return ws_root, ws_root / "tests-report" if os.path.isdir(ws_root / "bazel-testlogs"): - return ws_root / "bazel-testlogs" + return ws_root, ws_root / "bazel-testlogs" logger.info("could not find tests-report or bazel-testlogs to parse testcases") - return None + return ws_root, None def run_xml_parser(app: Sphinx, env: BuildEnvironment): @@ -203,11 +231,19 @@ def run_xml_parser(app: Sphinx, env: BuildEnvironment): building testcase needs. It gets called from the source_code_linker __init__ """ - testlogs_dir = find_test_folder() + root_path, testlogs_dir = find_test_folder() # early return if testlogs_dir is None: return xml_file_paths = find_xml_files(testlogs_dir) + # scl_with_metadata = load_source_code_links_with_metadata_json( + # app.outdir / "score_source_links_metadata.json" + # )[0] + # metadata: MetaData = { + # "module_name": scl_with_metadata.module_name, + # "hash": scl_with_metadata.hash, + # "url": scl_with_metadata.url, + # } test_case_needs = build_test_needs_from_files(app, env, xml_file_paths) # Saving the test case needs for cache store_data_of_test_case_json( @@ -262,6 +298,10 @@ def construct_and_add_need(app: Sphinx, tn: DataOfTestCase): # and either 'Fully' or 'PartiallyVerifies' should not be None here assert tn.file is not None assert tn.name is not None + assert tn.module_name is not None + assert tn.hash is not None + assert tn.url is not None + metadata = ModuleInfo(name=tn.module_name, hash=tn.hash, url=tn.url) # IDK if this is ideal or not with contextlib.suppress(BaseException): _ = add_external_need( @@ -271,7 +311,7 @@ def construct_and_add_need(app: Sphinx, tn: DataOfTestCase): tags="TEST", id=f"testcase__{tn.name}_{short_hash(tn.file + tn.name).upper()}", name=tn.name, - external_url=get_github_link(tn), + external_url=get_github_link(metadata, tn), fully_verifies=tn.FullyVerifies if tn.FullyVerifies is not None else "", partially_verifies=tn.PartiallyVerifies if tn.PartiallyVerifies is not None diff --git a/src/helper_lib/BUILD b/src/helper_lib/BUILD index 748a2a730..ad6316363 100644 --- a/src/helper_lib/BUILD +++ b/src/helper_lib/BUILD @@ -27,8 +27,7 @@ py_library( visibility = ["//visibility:public"], deps = [ "@rules_python//python/runfiles", - "@score_docs_as_code//src/extensions/score_source_code_linker:source_code_linker_helpers", - ], + ] + all_requirements, ) score_py_pytest( diff --git a/src/helper_lib/__init__.py b/src/helper_lib/__init__.py index 5699e478d..a72fffb0b 100644 --- a/src/helper_lib/__init__.py +++ b/src/helper_lib/__init__.py @@ -17,7 +17,7 @@ from pathlib import Path from typing import Any -from runfiles import Runfiles +from python.runfiles import Runfiles from sphinx.config import Config from sphinx_needs.logging import get_logger From c573b2f9f2d5130807b23da1a8ab65f63ec4b045 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Fri, 13 Mar 2026 16:52:42 +0100 Subject: [PATCH 11/34] Formatting --- scripts_bazel/generate_sourcelinks_cli.py | 1 - scripts_bazel/merge_sourcelinks.py | 2 +- src/extensions/score_source_code_linker/__init__.py | 2 +- .../score_source_code_linker/module_source_links.py | 1 - src/extensions/score_source_code_linker/testlink.py | 1 - .../score_source_code_linker/tests/test_codelink.py | 2 +- .../tests/test_source_code_link_integration.py | 3 +-- .../score_source_code_linker/xml_parser.py | 12 ++++++------ 8 files changed, 10 insertions(+), 14 deletions(-) diff --git a/scripts_bazel/generate_sourcelinks_cli.py b/scripts_bazel/generate_sourcelinks_cli.py index e1feb8e0b..6f09ef075 100644 --- a/scripts_bazel/generate_sourcelinks_cli.py +++ b/scripts_bazel/generate_sourcelinks_cli.py @@ -31,7 +31,6 @@ store_source_code_links_with_metadata_json, ) - logging.basicConfig(level=logging.INFO, format="%(message)s") logger = logging.getLogger(__name__) diff --git a/scripts_bazel/merge_sourcelinks.py b/scripts_bazel/merge_sourcelinks.py index afb2cc198..932d91380 100644 --- a/scripts_bazel/merge_sourcelinks.py +++ b/scripts_bazel/merge_sourcelinks.py @@ -85,7 +85,7 @@ def main(): for d in data[1:]: d.update(metadata) assert isinstance(data, list), repr(data) - merged.extend(data[1:]) + merged.extend(data[1:]) with open(args.output, "w") as f: json.dump(merged, f, indent=2, ensure_ascii=False) diff --git a/src/extensions/score_source_code_linker/__init__.py b/src/extensions/score_source_code_linker/__init__.py index c687e2e3e..d80754036 100644 --- a/src/extensions/score_source_code_linker/__init__.py +++ b/src/extensions/score_source_code_linker/__init__.py @@ -34,6 +34,7 @@ from src.extensions.score_source_code_linker.generate_source_code_links_json import ( generate_source_code_links_json, ) +from src.extensions.score_source_code_linker.helpers import get_github_link from src.extensions.score_source_code_linker.module_source_links import ( group_needs_by_module, load_module_source_links_json, @@ -60,7 +61,6 @@ find_git_root, find_ws_root, ) -from src.extensions.score_source_code_linker.helpers import get_github_link LOGGER = get_logger(__name__) # Uncomment this to enable more verbose logging diff --git a/src/extensions/score_source_code_linker/module_source_links.py b/src/extensions/score_source_code_linker/module_source_links.py index 97dc27a68..b77450eb8 100644 --- a/src/extensions/score_source_code_linker/module_source_links.py +++ b/src/extensions/score_source_code_linker/module_source_links.py @@ -15,7 +15,6 @@ import json from dataclasses import asdict, dataclass, field from pathlib import Path -from re import M from typing import Any from src.extensions.score_source_code_linker.need_source_links import ( diff --git a/src/extensions/score_source_code_linker/testlink.py b/src/extensions/score_source_code_linker/testlink.py index f001cc203..bd7139271 100644 --- a/src/extensions/score_source_code_linker/testlink.py +++ b/src/extensions/score_source_code_linker/testlink.py @@ -30,7 +30,6 @@ from sphinx_needs import logging - LOGGER = logging.get_logger(__name__) diff --git a/src/extensions/score_source_code_linker/tests/test_codelink.py b/src/extensions/score_source_code_linker/tests/test_codelink.py index 9615c0058..9d3fa78f5 100644 --- a/src/extensions/score_source_code_linker/tests/test_codelink.py +++ b/src/extensions/score_source_code_linker/tests/test_codelink.py @@ -35,6 +35,7 @@ get_cache_filename, group_by_need, ) +from src.extensions.score_source_code_linker.helpers import get_github_link from src.extensions.score_source_code_linker.needlinks import ( NeedLink, load_source_code_links_json, @@ -43,7 +44,6 @@ from src.helper_lib import ( get_current_git_hash, ) -from src.extensions.score_source_code_linker.helpers import get_github_link """ # ────────────────ATTENTION─────────────── diff --git a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py index 01bb7ff58..b4e65279d 100644 --- a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py +++ b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py @@ -25,6 +25,7 @@ from sphinx.testing.util import SphinxTestApp from sphinx_needs.data import SphinxNeedsData +from src.extensions.score_source_code_linker.helpers import get_github_link from src.extensions.score_source_code_linker.needlinks import NeedLink from src.extensions.score_source_code_linker.testlink import ( DataForTestLink, @@ -38,8 +39,6 @@ ) from src.helper_lib import find_ws_root, get_github_base_url -from src.extensions.score_source_code_linker.helpers import get_github_link - @pytest.fixture() def sphinx_base_dir(tmp_path_factory: TempPathFactory) -> Path: diff --git a/src/extensions/score_source_code_linker/xml_parser.py b/src/extensions/score_source_code_linker/xml_parser.py index 6facb983b..123ebd5d8 100644 --- a/src/extensions/score_source_code_linker/xml_parser.py +++ b/src/extensions/score_source_code_linker/xml_parser.py @@ -33,21 +33,21 @@ from sphinx_needs import logging from sphinx_needs.api import add_external_need +from src.extensions.score_source_code_linker.helpers import ( + get_github_link, + parse_info_from_known_good, + parse_module_name_from_path, +) +from src.extensions.score_source_code_linker.module_source_links import ModuleInfo from src.extensions.score_source_code_linker.needlinks import ( MetaData, ) -from src.extensions.score_source_code_linker.module_source_links import ModuleInfo from src.extensions.score_source_code_linker.testlink import ( DataOfTestCase, store_data_of_test_case_json, store_test_xml_parsed_json, ) from src.helper_lib import find_ws_root -from src.extensions.score_source_code_linker.helpers import ( - get_github_link, - parse_info_from_known_good, - parse_module_name_from_path, -) logger = logging.get_logger(__name__) logger.setLevel("DEBUG") From 9d87c2ccaffeb36433c4829702c5c3c04dc2a0c7 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Mon, 16 Mar 2026 11:00:31 +0100 Subject: [PATCH 12/34] Formating & Linting & Comments --- scripts_bazel/generate_sourcelinks_cli.py | 19 +++++++- scripts_bazel/merge_sourcelinks.py | 8 ---- .../generate_source_code_links_json.py | 8 +++- .../score_source_code_linker/xml_parser.py | 47 ++++++++++++------- 4 files changed, 54 insertions(+), 28 deletions(-) diff --git a/scripts_bazel/generate_sourcelinks_cli.py b/scripts_bazel/generate_sourcelinks_cli.py index 6f09ef075..89898d0ee 100644 --- a/scripts_bazel/generate_sourcelinks_cli.py +++ b/scripts_bazel/generate_sourcelinks_cli.py @@ -35,6 +35,22 @@ logger = logging.getLogger(__name__) +def clean_external_prefix(path: Path) -> Path: + """ + As it can be in combo builds that the path is: + `external/score_docs_as_code+/....` or similar + We have to remove this prefix from the file + before we pass it to the extract function. Otherwise we have + this prefix inside the `file` attribute leading to wrong links + """ + if not str(path).startswith("external/"): + return path + # This allows for files / folders etc. to have `external` in their name too. + path_raw = str(path).removeprefix("external/") + filepath_split = str(path_raw).split("/", maxsplit=1) + return Path("".join(filepath_split[1:])) + + def main(): parser = argparse.ArgumentParser( description="Generate source code links JSON from source files" @@ -67,8 +83,9 @@ def main(): metadata_set = True abs_file_path = file_path.resolve() assert abs_file_path.exists(), abs_file_path + clean_path = clean_external_prefix(file_path) references = _extract_references_from_file( - abs_file_path.parent, Path(abs_file_path.name), file_path + abs_file_path.parent, Path(abs_file_path.name), clean_path ) all_need_references.extend(references) store_source_code_links_with_metadata_json( diff --git a/scripts_bazel/merge_sourcelinks.py b/scripts_bazel/merge_sourcelinks.py index 932d91380..9325afc24 100644 --- a/scripts_bazel/merge_sourcelinks.py +++ b/scripts_bazel/merge_sourcelinks.py @@ -27,7 +27,6 @@ logger = logging.getLogger(__name__) - """ if bazel-out/k8-fastbuild/bin/external/ in file_path => module is external otherwise it's local @@ -35,12 +34,6 @@ if external => parse thing for module_name => look up known_good json for hash & url """ - - -def add_needid_to_metaneed_mapping(mapping: dict[str, dict[str, str]], metadata: dict[str, str], needid: str): - mapping - pass - def main(): parser = argparse.ArgumentParser( description="Merge multiple sourcelinks JSON files into one" @@ -67,7 +60,6 @@ def main(): all_files = [x for x in args.files if "known_good.json" not in str(x)] merged = [] - needs_metadata_mapping = {} for json_file in all_files: with open(json_file) as f: data = json.load(f) diff --git a/src/extensions/score_source_code_linker/generate_source_code_links_json.py b/src/extensions/score_source_code_linker/generate_source_code_links_json.py index 7b39b7228..8e9e2d807 100644 --- a/src/extensions/score_source_code_linker/generate_source_code_links_json.py +++ b/src/extensions/score_source_code_linker/generate_source_code_links_json.py @@ -50,7 +50,13 @@ def _extract_references_from_line(line: str): def _extract_references_from_file( root: Path, file_path_name: Path, file_path: Path ) -> list[NeedLink]: - """Scan a single file for template strings and return findings.""" + """Scan a single file for template strings and return findings. + Examples: + # ROOT: /docs-as-code/src/extensions/score_source_code_linker + #FILE PATH: + external/score_docs_as_code+/src/extensions/score_source_code_linker/testlink.py + #FILE PATH NAME: testlink.py + """ assert root.is_absolute(), "Root path must be absolute" assert not file_path_name.is_absolute(), "File path must be relative to the root" # assert file_path.is_relative_to(root), ( diff --git a/src/extensions/score_source_code_linker/xml_parser.py b/src/extensions/score_source_code_linker/xml_parser.py index 123ebd5d8..7347429ce 100644 --- a/src/extensions/score_source_code_linker/xml_parser.py +++ b/src/extensions/score_source_code_linker/xml_parser.py @@ -54,6 +54,23 @@ def get_metadata_from_test_path(filepath: Path) -> MetaData: + """ + Will parse out the metadata from the testpath. + If test is local then the metadata will be: + + "module_name": "local_module", + "hash": "", + "url": "", + + Else it will parse the module_name e.g. `score_docs_as_code` + match this in the known_good_json and grab the accompanying + hash, url as well and return metadata like so for example: + + "module_name": "score_docs_as_code", + "hash": "c1207676afe6cafd25c35d420e73279a799515d8", + "url": "https://github.com/eclipse-score/docs-as-code" + + """ known_good_json = os.environ.get("KNOWN_GOOD_JSON") module_name = parse_module_name_from_path(filepath) md: MetaData = { @@ -193,13 +210,17 @@ def read_test_xml_file(file: Path) -> tuple[list[DataOfTestCase], list[str], lis return test_case_needs, non_prop_tests, missing_prop_tests -# /home/maximilianp/score_personal/reference_integration/bazel-testlogs/external/score_docs_as_code+/src/helper_lib/helper_lib_tests/test.xml def find_xml_files(dir: Path) -> list[Path]: """ Recursively search all test.xml files inside 'bazel-testlogs' Returns: - list[Path] => Paths to all found 'test.xml' files. + + Example combo TestPath for future reference: + + '/reference_integration/bazel-testlogs + /feature_integration_tests/test_cases/fit/test.xml' """ test_file_name = "test.xml" @@ -208,21 +229,19 @@ def find_xml_files(dir: Path) -> list[Path]: for root, _, files in os.walk(dir): if test_file_name in files: xml_paths.append(Path(os.path.join(root, test_file_name))) - print("=========================================") - print(xml_paths[0]) - print("=========================================") + return xml_paths -def find_test_folder(base_path: Path | None = None) -> tuple[Path | None, Path | None]: +def find_test_folder(base_path: Path | None = None) -> Path | None: ws_root = base_path if base_path is not None else find_ws_root() assert ws_root is not None if os.path.isdir(ws_root / "tests-report"): - return ws_root, ws_root / "tests-report" + return ws_root / "tests-report" if os.path.isdir(ws_root / "bazel-testlogs"): - return ws_root, ws_root / "bazel-testlogs" + return ws_root / "bazel-testlogs" logger.info("could not find tests-report or bazel-testlogs to parse testcases") - return ws_root, None + return None def run_xml_parser(app: Sphinx, env: BuildEnvironment): @@ -231,19 +250,10 @@ def run_xml_parser(app: Sphinx, env: BuildEnvironment): building testcase needs. It gets called from the source_code_linker __init__ """ - root_path, testlogs_dir = find_test_folder() - # early return + testlogs_dir = find_test_folder() if testlogs_dir is None: return xml_file_paths = find_xml_files(testlogs_dir) - # scl_with_metadata = load_source_code_links_with_metadata_json( - # app.outdir / "score_source_links_metadata.json" - # )[0] - # metadata: MetaData = { - # "module_name": scl_with_metadata.module_name, - # "hash": scl_with_metadata.hash, - # "url": scl_with_metadata.url, - # } test_case_needs = build_test_needs_from_files(app, env, xml_file_paths) # Saving the test case needs for cache store_data_of_test_case_json( @@ -301,6 +311,7 @@ def construct_and_add_need(app: Sphinx, tn: DataOfTestCase): assert tn.module_name is not None assert tn.hash is not None assert tn.url is not None + # Have to build metadata here for the gh link func metadata = ModuleInfo(name=tn.module_name, hash=tn.hash, url=tn.url) # IDK if this is ideal or not with contextlib.suppress(BaseException): From f50ef0c0f57d2ea51fb790e3d543509351c380d6 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Mon, 16 Mar 2026 13:43:04 +0100 Subject: [PATCH 13/34] Fixing copilot & other findings --- docs.bzl | 73 ++++++++----------- scripts_bazel/merge_sourcelinks.py | 12 +++ src/extensions/score_source_code_linker/BUILD | 1 + .../score_source_code_linker/helpers.py | 18 ++--- .../module_source_links.py | 2 +- .../need_source_links.py | 3 - src/incremental.py | 2 + 7 files changed, 57 insertions(+), 54 deletions(-) diff --git a/docs.bzl b/docs.bzl index fe26590e8..9eca87261 100644 --- a/docs.bzl +++ b/docs.bzl @@ -183,33 +183,43 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good = data_with_docs_sources = _rewrite_needs_json_to_docs_sources(data) additional_combo_sourcelinks = _rewrite_needs_json_to_sourcelinks(data) _merge_sourcelinks(name = "merged_sourcelinks", sourcelinks = [":sourcelinks_json"] + additional_combo_sourcelinks, known_good = known_good) + docs_data = data + [":sourcelinks_json"] + combo_data = data_with_docs_sources + [":merged_sourcelinks"] + + docs_env = { + "SOURCE_DIRECTORY": source_dir, + "DATA": str(data), + "ACTION": "incremental", + "SCORE_SOURCELINKS": "$(location :sourcelinks_json)", + } + docs_sources_env = { + "SOURCE_DIRECTORY": source_dir, + "DATA": str(data_with_docs_sources), + "ACTION": "incremental", + "SCORE_SOURCELINKS": "$(location :merged_sourcelinks)", + } + if known_good: + docs_env["KNOWN_GOOD_JSON"] = "$(location "+ known_good + ")" + docs_sources_env["KNOWN_GOOD_JSON"] = "$(location "+ known_good + ")" + docs_data.append(known_good) + combo_data.append(known_good) py_binary( name = "docs", tags = ["cli_help=Build documentation:\nbazel run //:docs"], srcs = ["@score_docs_as_code//src:incremental.py"], - data = data + [":sourcelinks_json"], + data = docs_data, deps = deps, - env = { - "SOURCE_DIRECTORY": source_dir, - "DATA": str(data), - "ACTION": "incremental", - "SCORE_SOURCELINKS": "$(location :sourcelinks_json)", - }, + env = docs_env, ) py_binary( name = "docs_combo", tags = ["cli_help=Build full documentation with all dependencies:\nbazel run //:docs_combo"], srcs = ["@score_docs_as_code//src:incremental.py"], - data = data_with_docs_sources + [":merged_sourcelinks"], + data = combo_data, deps = deps, - env = { - "SOURCE_DIRECTORY": source_dir, - "DATA": str(data_with_docs_sources), - "ACTION": "incremental", - "SCORE_SOURCELINKS": "$(location :merged_sourcelinks)", - }, + env = docs_sources_env ) native.alias( @@ -222,55 +232,36 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good = name = "docs_link_check", tags = ["cli_help=Verify Links inside Documentation:\nbazel run //:link_check\n (Note: this could take a long time)"], srcs = ["@score_docs_as_code//src:incremental.py"], - data = data, + data = docs_data, deps = deps, - env = { - "SOURCE_DIRECTORY": source_dir, - "DATA": str(data), - "ACTION": "linkcheck", - }, + env = docs_env ) py_binary( name = "docs_check", tags = ["cli_help=Verify documentation:\nbazel run //:docs_check"], srcs = ["@score_docs_as_code//src:incremental.py"], - data = data + [":sourcelinks_json"], + data = docs_data, deps = deps, - env = { - "SOURCE_DIRECTORY": source_dir, - "DATA": str(data), - "ACTION": "check", - "SCORE_SOURCELINKS": "$(location :sourcelinks_json)", - }, + env = docs_env, ) py_binary( name = "live_preview", tags = ["cli_help=Live preview documentation in the browser:\nbazel run //:live_preview"], srcs = ["@score_docs_as_code//src:incremental.py"], - data = data + [":sourcelinks_json"], + data = docs_data, deps = deps, - env = { - "SOURCE_DIRECTORY": source_dir, - "DATA": str(data), - "ACTION": "live_preview", - "SCORE_SOURCELINKS": "$(location :sourcelinks_json)", - }, + env = docs_env, ) py_binary( name = "live_preview_combo_experimental", tags = ["cli_help=Live preview full documentation with all dependencies in the browser:\nbazel run //:live_preview_combo_experimental"], srcs = ["@score_docs_as_code//src:incremental.py"], - data = data_with_docs_sources + [":merged_sourcelinks"], + data = combo_data, deps = deps, - env = { - "SOURCE_DIRECTORY": source_dir, - "DATA": str(data_with_docs_sources), - "ACTION": "live_preview", - "SCORE_SOURCELINKS": "$(location :merged_sourcelinks)", - }, + env = docs_sources_env ) score_virtualenv( diff --git a/scripts_bazel/merge_sourcelinks.py b/scripts_bazel/merge_sourcelinks.py index 9325afc24..a6cc03e29 100644 --- a/scripts_bazel/merge_sourcelinks.py +++ b/scripts_bazel/merge_sourcelinks.py @@ -34,6 +34,7 @@ if external => parse thing for module_name => look up known_good json for hash & url """ + def main(): parser = argparse.ArgumentParser( description="Merge multiple sourcelinks JSON files into one" @@ -63,7 +64,18 @@ def main(): for json_file in all_files: with open(json_file) as f: data = json.load(f) + # If the file is empty e.g. '[]' there is nothing to parse, we continue + if not data: + continue metadata = data[0] + if not isinstance(metadata, dict) or "module_name" not in metadata: + logger.warning( + f"Unexpected schema in sourcelinks file '{json_file}': " + "expected first element to be a metadata dict " + "with a 'module_name' key. " + ) + # As we can't deal with bad JSON structure we just skip it + continue if metadata["module_name"] and metadata["module_name"] != "local_module": hash, repo = parse_info_from_known_good( known_good_json=args.known_good, module_name=metadata["module_name"] diff --git a/src/extensions/score_source_code_linker/BUILD b/src/extensions/score_source_code_linker/BUILD index 55d471374..85b8f222b 100644 --- a/src/extensions/score_source_code_linker/BUILD +++ b/src/extensions/score_source_code_linker/BUILD @@ -55,6 +55,7 @@ py_library( "testlink.py", "xml_parser.py", "helpers.py", + "module_source_links.py", ], imports = ["."], visibility = ["//visibility:public"], diff --git a/src/extensions/score_source_code_linker/helpers.py b/src/extensions/score_source_code_linker/helpers.py index 5544e2696..ad471f757 100644 --- a/src/extensions/score_source_code_linker/helpers.py +++ b/src/extensions/score_source_code_linker/helpers.py @@ -57,7 +57,7 @@ def get_github_link_from_git( def get_github_link_from_json( metadata: ModuleInfo, link: NeedLink | DataForTestLink | DataOfTestCase | None = None, -): +) -> str: if link is None: link = DefaultNeedLink() base_url = metadata.url @@ -67,16 +67,17 @@ def get_github_link_from_json( def parse_module_name_from_path(path: Path) -> str: """ - Parse out the Module-Name from the filename gotten - /home/user/.cache/bazel/aksj37981712/external/score_docs_as_code+/src/tests/testfile.py - => score_docs_as_code + Parse out the Module-Name from the filename: + Combo Example: + Path: external/score_docs_as_code+/src/helper_lib/test_helper_lib.py + => score_docs_as_code + Local: + Path: src/helper_lib/test_helper_lib.py + => local_module + """ # COMBO BUILD - # If external is in the filepath that gets parsed => - # file is in an external module => combo build - # Example Path: - # PosixPath('external/score_docs_as_code+/src/helper_lib/test_helper_lib.py' if str(path).startswith("external/"): # This allows for files / folders etc. to have `external` in their name too. @@ -84,7 +85,6 @@ def parse_module_name_from_path(path: Path) -> str: filepath_split = str(module_raw).split("/", maxsplit=1) return str(filepath_split[0].removesuffix("+")) # We return this when we are in a local build `//:docs` the rest of DaC knows - # What to do then if it encounters this module_name return "local_module" diff --git a/src/extensions/score_source_code_linker/module_source_links.py b/src/extensions/score_source_code_linker/module_source_links.py index b77450eb8..bf299a49f 100644 --- a/src/extensions/score_source_code_linker/module_source_links.py +++ b/src/extensions/score_source_code_linker/module_source_links.py @@ -78,7 +78,7 @@ def store_module_source_links_json( # After `rm -rf _build` or on clean builds the directory does not exist, so we need # to create it file.parent.mkdir(exist_ok=True) - with open(file, "w") as f: + with open(file, "w", encoding="utf-8") as f: json.dump( source_code_links, f, diff --git a/src/extensions/score_source_code_linker/need_source_links.py b/src/extensions/score_source_code_linker/need_source_links.py index 1823ba4b4..1582c2fe8 100644 --- a/src/extensions/score_source_code_linker/need_source_links.py +++ b/src/extensions/score_source_code_linker/need_source_links.py @@ -123,9 +123,6 @@ def group_by_need( { "need": "", - "module_name": , - "hash": , - "url": , "links": { "CodeLinks": [NeedLink, NeedLink, ...], "TestLinks": [testlink, testlink, ...] diff --git a/src/incremental.py b/src/incremental.py index 1c3816229..fa1c5abc2 100644 --- a/src/incremental.py +++ b/src/incremental.py @@ -90,6 +90,8 @@ def get_env(name: str) -> str: base_arguments.append(f"-A=github_repo={args.github_repo}") base_arguments.append("-A=github_version=main") base_arguments.append(f"-A=doc_path={get_env('SOURCE_DIRECTORY')}") + if os.getenv("KNOWN_GOOD_JSON"): + base_arguments.append(f"--define=KNOWN_GOOD_JSON={get_env('KNOWN_GOOD_JSON')}") action = get_env("ACTION") if action == "live_preview": From 976c2eccf60e1cdd6732babdb36c1347eb24b93e Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Mon, 16 Mar 2026 20:03:01 +0100 Subject: [PATCH 14/34] WIP: Tests debuging --- pyproject.toml | 2 + .../generate_source_code_links_json.py | 6 +++ .../tests/expected_codelink.json | 20 +++++++-- .../tests/expected_grouped.json | 45 +++++++++++++++---- .../tests/expected_testlink.json | 25 ++++++++--- .../tests/test_codelink.py | 20 ++++++--- .../test_source_code_link_integration.py | 2 +- .../tests/test_testlink.py | 22 ++++++++- .../tests/test_xml_parser.py | 18 +++++++- 9 files changed, 133 insertions(+), 27 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6aa48c6f3..b9efbffb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ extend-exclude = [ ".venv*/**", ] [tool.pytest.ini_options] +addopts = ["-v", "-s"] log_cli = true log_cli_level = "Debug" log_cli_format = "[%(asctime)s.%(msecs)03d] [%(levelname)-3s] [%(name)s] %(message)s" @@ -66,3 +67,4 @@ filterwarnings = [ pythonpath = [ "src/extensions/", ] +basetemp = "./debug_tmp" diff --git a/src/extensions/score_source_code_linker/generate_source_code_links_json.py b/src/extensions/score_source_code_linker/generate_source_code_links_json.py index 8e9e2d807..44c1c8bed 100644 --- a/src/extensions/score_source_code_linker/generate_source_code_links_json.py +++ b/src/extensions/score_source_code_linker/generate_source_code_links_json.py @@ -57,6 +57,9 @@ def _extract_references_from_file( external/score_docs_as_code+/src/extensions/score_source_code_linker/testlink.py #FILE PATH NAME: testlink.py """ + print("ROOT: ", root) + print("FILE_PATH: ", file_path) + print("FILE_PATH_NAME: ", file_path_name) assert root.is_absolute(), "Root path must be absolute" assert not file_path_name.is_absolute(), "File path must be relative to the root" # assert file_path.is_relative_to(root), ( @@ -130,6 +133,9 @@ def find_all_need_references(search_path: Path) -> list[NeedLink]: # Use os.walk to have better control over directory traversal for file in iterate_files_recursively(search_path): + print("Search_path: ", search_path) + print("File.name: ", file.name) + print("File: ", file) references = _extract_references_from_file(search_path,Path(file.name), file) all_need_references.extend(references) diff --git a/src/extensions/score_source_code_linker/tests/expected_codelink.json b/src/extensions/score_source_code_linker/tests/expected_codelink.json index 447ef8a69..4ff1f14bb 100644 --- a/src/extensions/score_source_code_linker/tests/expected_codelink.json +++ b/src/extensions/score_source_code_linker/tests/expected_codelink.json @@ -4,27 +4,39 @@ "line": 3, "tag":"#-----req-Id:", "need": "TREQ_ID_1", - "full_line": "#-----req-Id: TREQ_ID_1" + "full_line": "#-----req-Id: TREQ_ID_1", + "module_name": "local_module", + "hash": "", + "url": "" }, { "file": "src/implementation2.py", "line": 5, "tag":"#-----req-Id:", "need": "TREQ_ID_1", - "full_line": "#-----req-Id: TREQ_ID_1" + "full_line": "#-----req-Id: TREQ_ID_1", + "module_name": "local_module", + "hash": "", + "url": "" }, { "file": "src/implementation1.py", "line": 9, "tag":"#-----req-Id:", "need": "TREQ_ID_2", - "full_line":"#-----req-Id: TREQ_ID_2" + "full_line":"#-----req-Id: TREQ_ID_2", + "module_name": "local_module", + "hash": "", + "url": "" }, { "file": "src/bad_implementation.py", "line":2, "tag":"#-----req-Id:", "need": "TREQ_ID_200", - "full_line":"#-----req-Id: TREQ_ID_200" + "full_line":"#-----req-Id: TREQ_ID_200", + "module_name": "local_module", + "hash": "", + "url": "" } ] diff --git a/src/extensions/score_source_code_linker/tests/expected_grouped.json b/src/extensions/score_source_code_linker/tests/expected_grouped.json index 256232661..bd6df7e40 100644 --- a/src/extensions/score_source_code_linker/tests/expected_grouped.json +++ b/src/extensions/score_source_code_linker/tests/expected_grouped.json @@ -8,14 +8,20 @@ "line": 3, "tag":"#-----req-Id:", "need": "TREQ_ID_1", - "full_line": "#-----req-Id: TREQ_ID_1" + "full_line": "#-----req-Id: TREQ_ID_1", + "module_name": "local_module", + "hash": "", + "url": "" }, { "file": "src/implementation2.py", "line": 5, "tag":"#-----req-Id:", "need": "TREQ_ID_1", - "full_line": "#-----req-Id: TREQ_ID_1" + "full_line": "#-----req-Id: TREQ_ID_1", + "module_name": "local_module", + "hash": "", + "url": "" } ], @@ -27,7 +33,10 @@ "need": "TREQ_ID_1", "verify_type": "fully", "result": "passed", - "result_text": "" + "result_text": "", + "module_name": "local_module", + "hash": "", + "url": "" } ] } @@ -41,7 +50,10 @@ "line": 9, "tag":"#-----req-Id:", "need": "TREQ_ID_2", - "full_line":"#-----req-Id: TREQ_ID_2" + "full_line":"#-----req-Id: TREQ_ID_2", + "module_name": "local_module", + "hash": "", + "url": "" } ], "TestLinks": [ @@ -53,7 +65,10 @@ "need": "TREQ_ID_2", "verify_type": "partially", "result": "passed", - "result_text": "" + "result_text": "", + "module_name": "local_module", + "hash": "", + "url": "" }, { "name": "test_error_handling", @@ -62,7 +77,10 @@ "need": "TREQ_ID_2", "verify_type": "partially", "result": "passed", - "result_text": "" + "result_text": "", + "module_name": "local_module", + "hash": "", + "url": "" } ] @@ -80,7 +98,10 @@ "need": "TREQ_ID_3", "verify_type": "partially", "result": "passed", - "result_text": "" + "result_text": "", + "module_name": "local_module", + "hash": "", + "url": "" }, { "name": "test_error_handling", @@ -89,7 +110,10 @@ "need": "TREQ_ID_3", "verify_type": "partially", "result": "passed", - "result_text": "" + "result_text": "", + "module_name": "local_module", + "hash": "", + "url": "" } ] } @@ -103,7 +127,10 @@ "line":2, "tag":"#-----req-Id:", "need": "TREQ_ID_200", - "full_line":"#-----req-Id: TREQ_ID_200" + "full_line":"#-----req-Id: TREQ_ID_200", + "module_name": "local_module", + "hash": "", + "url": "" } ], diff --git a/src/extensions/score_source_code_linker/tests/expected_testlink.json b/src/extensions/score_source_code_linker/tests/expected_testlink.json index 19068a4d5..145c31c31 100644 --- a/src/extensions/score_source_code_linker/tests/expected_testlink.json +++ b/src/extensions/score_source_code_linker/tests/expected_testlink.json @@ -6,7 +6,10 @@ "need": "TREQ_ID_2", "verify_type": "partially", "result": "passed", - "result_text": "" + "result_text": "", + "module_name": "local_module", + "hash": "", + "url": "" }, { "name": "test_api_response_format", @@ -15,7 +18,10 @@ "need": "TREQ_ID_3", "verify_type": "partially", "result": "passed", - "result_text": "" + "result_text": "", + "module_name": "local_module", + "hash": "", + "url": "" }, { "name": "test_error_handling", @@ -24,7 +30,10 @@ "need": "TREQ_ID_2", "verify_type": "partially", "result": "passed", - "result_text": "" + "result_text": "", + "module_name": "local_module", + "hash": "", + "url": "" }, { "name": "test_error_handling", @@ -33,7 +42,10 @@ "need": "TREQ_ID_3", "verify_type": "partially", "result": "passed", - "result_text": "" + "result_text": "", + "module_name": "local_module", + "hash": "", + "url": "" }, { "name": "TestRequirementsCoverage__test_system_startup_time", @@ -42,6 +54,9 @@ "need": "TREQ_ID_1", "verify_type": "fully", "result": "passed", - "result_text": "" + "result_text": "", + "module_name": "local_module", + "hash": "", + "url": "" } ] diff --git a/src/extensions/score_source_code_linker/tests/test_codelink.py b/src/extensions/score_source_code_linker/tests/test_codelink.py index 9d3fa78f5..ff23ad184 100644 --- a/src/extensions/score_source_code_linker/tests/test_codelink.py +++ b/src/extensions/score_source_code_linker/tests/test_codelink.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2025 Contributors to the Eclipse Foundation +# Copyright (c) 2025-2026 Contributors to the Eclipse Foundation # # See the NOTICE file(s) distributed with this work for additional # information regarding copyright ownership. @@ -35,7 +35,6 @@ get_cache_filename, group_by_need, ) -from src.extensions.score_source_code_linker.helpers import get_github_link from src.extensions.score_source_code_linker.needlinks import ( NeedLink, load_source_code_links_json, @@ -44,6 +43,10 @@ from src.helper_lib import ( get_current_git_hash, ) +# ADAPTED: Importing from the new generator location +from src.extensions.score_source_code_linker.helpers import ( + get_github_link_from_git, +) """ # ────────────────ATTENTION─────────────── @@ -127,6 +130,7 @@ def default(self, o: object): def needlink_test_decoder(d: dict[str, Any]) -> NeedLink | dict[str, Any]: + # ADAPTED: Updated to include new optional fields (module_name, hash, url) if {"file", "line", "tag", "need", "full_line"} <= d.keys(): return NeedLink( file=Path(d["file"]), @@ -134,6 +138,9 @@ def needlink_test_decoder(d: dict[str, Any]) -> NeedLink | dict[str, Any]: tag=decode_comment(d["tag"]), need=d["need"], full_line=decode_comment(d["full_line"]), + module_name=d.get("module_name", ""), + hash=d.get("hash", ""), + url=d.get("url", ""), ) # It's something else, pass it on to other decoders return d @@ -347,7 +354,8 @@ def test_get_github_link_with_real_repo(git_repo: Path) -> None: # Have to change directories in order to ensure that we get the right/any .git file os.chdir(Path(git_repo).absolute()) - result = get_github_link(needlink) + # ADAPTED: Using get_github_link_from_git for direct local repo testing + result = get_github_link_from_git(needlink) # Should contain the base URL, hash, file path, and line number assert "https://github.com/test-user/test-repo/blob/" in result @@ -549,7 +557,8 @@ def another_function(): # Have to change directories in order to ensure that we get the right/any .git file os.chdir(Path(git_repo).absolute()) for needlink in loaded_links: - github_link = get_github_link(needlink) + # ADAPTED: Using get_github_link_from_git + github_link = get_github_link_from_git(needlink) assert "https://github.com/test-user/test-repo/blob/" in github_link assert f"src/{needlink.file.name}#L{needlink.line}" in github_link @@ -587,5 +596,6 @@ def test_multiple_commits_hash_consistency(git_repo: Path) -> None: ) os.chdir(Path(git_repo).absolute()) - github_link = get_github_link(needlink) + # ADAPTED: Using get_github_link_from_git + github_link = get_github_link_from_git(needlink) assert new_hash in github_link diff --git a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py index b4e65279d..fb4780dc1 100644 --- a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py +++ b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py @@ -25,7 +25,6 @@ from sphinx.testing.util import SphinxTestApp from sphinx_needs.data import SphinxNeedsData -from src.extensions.score_source_code_linker.helpers import get_github_link from src.extensions.score_source_code_linker.needlinks import NeedLink from src.extensions.score_source_code_linker.testlink import ( DataForTestLink, @@ -38,6 +37,7 @@ SourceCodeLinks_TEST_JSON_Decoder, ) from src.helper_lib import find_ws_root, get_github_base_url +from src.extensions.score_source_code_linker.helpers import get_github_link @pytest.fixture() diff --git a/src/extensions/score_source_code_linker/tests/test_testlink.py b/src/extensions/score_source_code_linker/tests/test_testlink.py index 74becef6b..de6811183 100644 --- a/src/extensions/score_source_code_linker/tests/test_testlink.py +++ b/src/extensions/score_source_code_linker/tests/test_testlink.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2025 Contributors to the Eclipse Foundation +# Copyright (c) 2025-2026 Contributors to the Eclipse Foundation # # See the NOTICE file(s) distributed with this work for additional # information regarding copyright ownership. @@ -41,6 +41,10 @@ def test_testlink_serialization_roundtrip(): verify_type="fully", result="passed", result_text="All good", + # ADAPTED: Added new fields + module_name="test_module", + hash="abc12345", + url="https://github.com/org/repo", ) dumped = json.dumps(link, cls=DataForTestLink_JSON_Encoder) loaded = json.loads(dumped, object_hook=DataForTestLink_JSON_Decoder) @@ -101,6 +105,10 @@ def test_testcaseneed_to_dict_multiple_links(): result_text="Something went wrong", PartiallyVerifies="REQ-1, REQ-2", FullyVerifies="REQ-3", + # ADAPTED: Added new fields which are now required for valid TestLinks + module_name="test_module", + hash="hash123", + url="http://github.com", ) links = case.get_test_links() @@ -114,6 +122,10 @@ def test_testcaseneed_to_dict_multiple_links(): assert link.line == 10 assert link.name == "TC_01" assert link.result == "failed" + # ADAPTED: Verify new fields are propagated + assert link.module_name == "test_module" + assert link.hash == "hash123" + assert link.url == "http://github.com" @add_test_properties( @@ -134,6 +146,10 @@ def test_store_and_load_testlinks_roundtrip(tmp_path: Path): verify_type="partially", result="passed", result_text="Looks good", + # ADAPTED: Added new fields + module_name="mod1", + hash="h1", + url="u1", ), DataForTestLink( name="L2", @@ -143,6 +159,10 @@ def test_store_and_load_testlinks_roundtrip(tmp_path: Path): verify_type="fully", result="failed", result_text="Needs work", + # ADAPTED: Added new fields + module_name="mod2", + hash="h2", + url="u2", ), ] diff --git a/src/extensions/score_source_code_linker/tests/test_xml_parser.py b/src/extensions/score_source_code_linker/tests/test_xml_parser.py index 95b445dde..c283aada5 100644 --- a/src/extensions/score_source_code_linker/tests/test_xml_parser.py +++ b/src/extensions/score_source_code_linker/tests/test_xml_parser.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2025 Contributors to the Eclipse Foundation +# Copyright (c) 2025-2026 Contributors to the Eclipse Foundation # # See the NOTICE file(s) distributed with this work for additional # information regarding copyright ownership. @@ -20,6 +20,7 @@ from collections.abc import Callable from pathlib import Path from typing import Any +from unittest.mock import patch import pytest @@ -262,10 +263,18 @@ def test_parse_properties(): test_type="requirements-based", derivation_technique="requirements-analysis", ) +# ADAPTED: Added patching for metadata functions +@patch("src.extensions.score_source_code_linker.xml_parser.parse_module_name_from_path") def test_read_test_xml_file( + mock_parse_module: Any, tmp_xml_dirs: Callable[..., tuple[Path, Path, Path, Path, Path]], ): """Ensure a whole pre-defined xml file is parsed correctly""" + # ADAPTED: Mock return value to ensure metadata is populated. + # 'local_module' triggers the path where hash/url are empty strings, + # which is valid but avoids the need to mock parse_info_from_known_good. + mock_parse_module.return_value = "local_module" + _: Path dir1: Path dir2: Path @@ -276,6 +285,11 @@ def test_read_test_xml_file( tcneed = needs1[0] assert isinstance(tcneed, DataOfTestCase) assert tcneed.result == "failed" + # ADAPTED: Verify metadata fields were populated + assert tcneed.module_name == "local_module" + assert tcneed.hash == "" + assert tcneed.url == "" + assert no_props1 == [] assert missing_props1 == [] @@ -287,7 +301,7 @@ def test_read_test_xml_file( # Extra Properties => Should not cause an error needs3, no_props3, missing_props3 = xml_parser.read_test_xml_file(dir3 / "test.xml") - assert isinstance(needs1, list) and len(needs1) == 1 + assert isinstance(needs3, list) and len(needs3) == 1 tcneed3 = needs3[0] assert isinstance(tcneed3, DataOfTestCase) assert no_props3 == [] From d094914cec1a1c9a8c76ee693ce2dba67dec5950 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Tue, 17 Mar 2026 05:32:23 +0100 Subject: [PATCH 15/34] Tests Passing & Tests adapted --- pyproject.toml | 1 - .../generate_source_code_links_json.py | 5 +- .../score_source_code_linker/helpers.py | 12 + .../score_source_code_linker/needlinks.py | 36 +- .../score_source_code_linker/testlink.py | 41 ++- .../tests/test_codelink.py | 32 +- .../tests/test_helpers.py | 347 ++++++++++++++++++ .../test_source_code_link_integration.py | 45 ++- 8 files changed, 478 insertions(+), 41 deletions(-) create mode 100644 src/extensions/score_source_code_linker/tests/test_helpers.py diff --git a/pyproject.toml b/pyproject.toml index b9efbffb8..8dd24d16b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,4 +67,3 @@ filterwarnings = [ pythonpath = [ "src/extensions/", ] -basetemp = "./debug_tmp" diff --git a/src/extensions/score_source_code_linker/generate_source_code_links_json.py b/src/extensions/score_source_code_linker/generate_source_code_links_json.py index 44c1c8bed..87e4aa554 100644 --- a/src/extensions/score_source_code_linker/generate_source_code_links_json.py +++ b/src/extensions/score_source_code_linker/generate_source_code_links_json.py @@ -19,6 +19,7 @@ import os from pathlib import Path +import subprocess from sphinx_needs.logging import get_logger @@ -60,6 +61,8 @@ def _extract_references_from_file( print("ROOT: ", root) print("FILE_PATH: ", file_path) print("FILE_PATH_NAME: ", file_path_name) + output = subprocess.run(["ls", "-la", f"{root}"], capture_output=True) + print(output) assert root.is_absolute(), "Root path must be absolute" assert not file_path_name.is_absolute(), "File path must be relative to the root" # assert file_path.is_relative_to(root), ( @@ -136,7 +139,7 @@ def find_all_need_references(search_path: Path) -> list[NeedLink]: print("Search_path: ", search_path) print("File.name: ", file.name) print("File: ", file) - references = _extract_references_from_file(search_path,Path(file.name), file) + references = _extract_references_from_file(search_path,Path(file), file) all_need_references.extend(references) elapsed_time = os.times().elapsed - start_time diff --git a/src/extensions/score_source_code_linker/helpers.py b/src/extensions/score_source_code_linker/helpers.py index ad471f757..a186754ac 100644 --- a/src/extensions/score_source_code_linker/helpers.py +++ b/src/extensions/score_source_code_linker/helpers.py @@ -93,6 +93,18 @@ def parse_info_from_known_good( ) -> tuple[str, str]: with open(known_good_json) as f: kg_json = json.load(f) + + # ───────[ Assert our worldview that has to exist here ]───── + assert kg_json, ( + f"Known good json at: {known_good_json} is empty. This is not allowed" + ) + assert "modules" in kg_json, ( + f"Known good json at: {known_good_json} is missing the 'modules' key" + ) + assert kg_json["modules"], ( + f"Known good json at: {known_good_json} has an empty 'modules' dictionary" + ) + for category in kg_json["modules"].values(): if module_name in category: m = category[module_name] diff --git a/src/extensions/score_source_code_linker/needlinks.py b/src/extensions/score_source_code_linker/needlinks.py index 8bc8f9439..0f55d6dd3 100644 --- a/src/extensions/score_source_code_linker/needlinks.py +++ b/src/extensions/score_source_code_linker/needlinks.py @@ -24,6 +24,7 @@ class MetaData(TypedDict): hash: str url: str + def is_metadata(x: object) -> TypeGuard[MetaData]: # Make this as strict/loose as you want; at minimum, it must be a dict. return isinstance(x, dict) and {"module_name", "hash", "url"} <= x.keys() @@ -38,13 +39,46 @@ class NeedLink: tag: str need: str full_line: str - module_name: str = "" + module_name: str = "local_module" hash: str = "" url: str = "" + # Adding hashing & equality as this is needed to make comparisions. + # Since the Dataclass is not 'frozen = true' it isn't automatically hashable + def __hash__(self): + return hash( + ( + self.file, + self.line, + self.tag, + self.need, + self.full_line, + self.module_name, + self.hash, + self.url, + ) + ) + + def __eq__(self, other): + if not isinstance(other, NeedLink): + return NotImplemented + return ( + self.file == other.file + and self.line == other.line + and self.tag == other.tag + and self.need == other.need + and self.full_line == other.full_line + and self.module_name == other.module_name + and self.hash == other.hash + and self.url == other.url + ) + + # Normal 'dictionary conversion'. Converts all fields def to_dict_full(self) -> dict[str, str | Path | int]: return asdict(self) + # Drops MetaData fields for saving the Dataclass (saving space in json) + # The information is in the 'Module_Source_Link' in the end def to_dict_without_metadata(self) -> dict[str, str | Path | int]: d = asdict(self) d.pop("module_name", None) diff --git a/src/extensions/score_source_code_linker/testlink.py b/src/extensions/score_source_code_linker/testlink.py index bd7139271..170bd0ad4 100644 --- a/src/extensions/score_source_code_linker/testlink.py +++ b/src/extensions/score_source_code_linker/testlink.py @@ -33,7 +33,7 @@ LOGGER = logging.get_logger(__name__) -@dataclass(frozen=True, order=True) +@dataclass(order=True) class DataForTestLink: name: str file: Path @@ -42,13 +42,50 @@ class DataForTestLink: verify_type: str result: str result_text: str = "" - module_name: str = "" + module_name: str = "local_module" hash: str = "" url: str = "" + # Adding hashing & equality as this is needed to make comparisions. + # Since the Dataclass is not 'frozen = true' it isn't automatically hashable + def __hash__(self): + return hash( + ( + self.name, + str(self.file), + self.line, + self.need, + self.verify_type, + self.result, + self.result_text, + self.module_name, + self.hash, + self.url, + ) + ) + + def __eq__(self, other): + if not isinstance(other, DataForTestLink): + return NotImplemented + return ( + self.name == other.name + and self.file == other.file + and self.line == other.line + and self.need == other.need + and self.verify_type == other.verify_type + and self.result == other.result + and self.result_text == other.result_text + and self.module_name == other.module_name + and self.hash == other.hash + and self.url == other.url + ) + + # Normal 'dictionary conversion'. Converts all fields def to_dict_full(self) -> dict[str, str | Path | int]: return asdict(self) + # Drops MetaData fields for saving the Dataclass (saving space in json) + # The information is in the 'Module_Source_Link' in the end def to_dict_without_metadata(self) -> dict[str, str | Path | int]: d = asdict(self) d.pop("module_name", None) diff --git a/src/extensions/score_source_code_linker/tests/test_codelink.py b/src/extensions/score_source_code_linker/tests/test_codelink.py index ff23ad184..46d45a59c 100644 --- a/src/extensions/score_source_code_linker/tests/test_codelink.py +++ b/src/extensions/score_source_code_linker/tests/test_codelink.py @@ -131,7 +131,7 @@ def default(self, o: object): def needlink_test_decoder(d: dict[str, Any]) -> NeedLink | dict[str, Any]: # ADAPTED: Updated to include new optional fields (module_name, hash, url) - if {"file", "line", "tag", "need", "full_line"} <= d.keys(): + if {"file", "line", "tag", "need", "full_line", "module_name", "hash" ,"url" } <= d.keys(): return NeedLink( file=Path(d["file"]), line=d["line"], @@ -336,36 +336,6 @@ def test_group_by_need_empty_list(): assert len(result) == 0 -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) -def test_get_github_link_with_real_repo(git_repo: Path) -> None: - """Test generating GitHub link with real repository.""" - # Create a needlink - needlink = NeedLink( - file=Path("src/test.py"), - line=42, - tag="#" + " req-Id:", - need="REQ_001", - full_line="#" + " req-Id: REQ_001", - ) - - # Have to change directories in order to ensure that we get the right/any .git file - os.chdir(Path(git_repo).absolute()) - # ADAPTED: Using get_github_link_from_git for direct local repo testing - result = get_github_link_from_git(needlink) - - # Should contain the base URL, hash, file path, and line number - assert "https://github.com/test-user/test-repo/blob/" in result - assert "src/test.py#L42" in result - assert len(result.split("/")) >= 7 # Should have proper URL structure - - # Verify the hash is actually from the repo - hash_from_link = result.split("/blob/")[1].split("/")[0] - actual_hash = get_current_git_hash(git_repo) - assert hash_from_link == actual_hash # Test cache file operations diff --git a/src/extensions/score_source_code_linker/tests/test_helpers.py b/src/extensions/score_source_code_linker/tests/test_helpers.py new file mode 100644 index 000000000..593253583 --- /dev/null +++ b/src/extensions/score_source_code_linker/tests/test_helpers.py @@ -0,0 +1,347 @@ +# ******************************************************************************* +# Copyright (c) 2025 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import json +import os +from pathlib import Path +import tempfile +from collections.abc import Generator + +import pytest +import subprocess + +from src.helper_lib import get_current_git_hash +from src.extensions.score_source_code_linker.module_source_links import ModuleInfo +from src.extensions.score_source_code_linker.needlinks import DefaultNeedLink + +from src.extensions.score_source_code_linker.helpers import ( + get_github_link, + get_github_link_from_json, + parse_info_from_known_good, + parse_module_name_from_path, +) + + +# ╭──────────────────────────────────────────────────────────╮ +# │ Tests for parse_module_name_from_path │ +# ╰──────────────────────────────────────────────────────────╯ + + +def test_parse_module_name_from_external_path(): + """Test parsing module name from external/combo build path.""" + path = Path("external/score_docs_as_code+/src/helper_lib/test_helper_lib.py") + + result = parse_module_name_from_path(path) + + assert result == "score_docs_as_code" + + +def test_parse_module_name_from_external_path_2(): + """Test that an extra 'external' in the path does not change the output""" + path = Path( + "external/score_docs_as_code+/src/external/helper_lib/test_helper_lib.py" + ) + + result = parse_module_name_from_path(path) + + assert result == "score_docs_as_code" + + +def test_parse_module_name_from_local_path(): + """Test parsing module name from local build path.""" + path = Path("src/helper_lib/test_helper_lib.py") + + result = parse_module_name_from_path(path) + + assert result == "local_module" + + +def test_parse_module_name_from_empty_path(): + """Test parsing module name from an empty path.""" + path = Path("") + + result = parse_module_name_from_path(path) + + assert result == "local_module" + + +def test_parse_module_name_without_plus_suffix(): + """ + Test parsing external module without plus suffix. + This should never happen (due to bazel adding the '+') + But testing this edge case anyway + """ + path = Path("external/module_without_plus/file.py") + + result = parse_module_name_from_path(path) + + assert result == "module_without_plus" + + +# ╭──────────────────────────────────────────────────────────╮ +# │ PARSE INFO FROM KNOWN GOOD │ +# ╰──────────────────────────────────────────────────────────╯ + + +# Example of a minimal valid known_good.json structure +VALID_KNOWN_GOOD = { + "modules": { + "target_sw": { + "score_baselibs": { + "repo": "https://github.com/eclipse-score/baselibs.git", + "hash": "158fe6a7b791c58f6eac5f7e4662b8db0cf9ac6e", + }, + "score_communication": { + "repo": "https://github.com/eclipse-score/communication.git", + "hash": "56448a5589a5f7d3921b873e8127b824a8c1ca95", + }, + }, + "tooling": { + "score_docs_as_code": { + "repo": "https://github.com/eclipse-score/docs-as-code.git", + "hash": "c1207676afe6cafd25c35d420e73279a799515d8", + } + }, + } +} + +# ──────────────────────────────────────────────────────────── +# ──────────────────────[ Test Functions ]────────────────────── + + +@pytest.fixture +def known_good_json(tmp_path: Path): + """Fixture providing a valid known_good.json file.""" + json_file = tmp_path / "known_good.json" + _ = json_file.write_text(json.dumps(VALID_KNOWN_GOOD)) + return json_file + + +# Tests for parse_info_from_known_good +def test_parse_info_from_known_good_happy_path(known_good_json: Path): + """Test parsing module info from valid known_good.json.""" + hash_result, repo_result = parse_info_from_known_good( + known_good_json, "score_baselibs" + ) + + assert hash_result == "158fe6a7b791c58f6eac5f7e4662b8db0cf9ac6e" + assert repo_result == "https://github.com/eclipse-score/baselibs" + + +def test_parse_info_from_known_good_different_category(known_good_json: Path): + """Test finding module in different category.""" + hash_result, repo_result = parse_info_from_known_good( + known_good_json, "score_docs_as_code" + ) + + assert hash_result == "c1207676afe6cafd25c35d420e73279a799515d8" + assert repo_result == "https://github.com/eclipse-score/docs-as-code" + + +def test_parse_info_from_known_good_module_not_found(known_good_json: Path): + """Test that KeyError is raised when module doesn't exist.""" + with pytest.raises(KeyError, match="Module 'nonexistent' not found"): + parse_info_from_known_good(known_good_json, "nonexistent") + + +def test_parse_info_from_known_good_empty_json(tmp_path: Path): + """Test with empty JSON file.""" + json_file = tmp_path / "example.json" + _ = json_file.write_text("{}") + + with pytest.raises( + AssertionError, + match=f"Known good json at: {json_file} is empty. This is not allowed", + ): + parse_info_from_known_good(json_file, "any_module") + + +def test_parse_info_from_known_good_no_module_in_json(tmp_path: Path): + """Test that assertion works when module not in top level keys""" + json_file = tmp_path / "example.json" + _ = json_file.write_text( + '{"another_key": {"a": "b"}, "second_key": ["a" , "b", "c"]}' + ) + + with pytest.raises( + AssertionError, + match=f"Known good json at: {json_file} is missing the 'modules' key", + ): + parse_info_from_known_good(json_file, "any_module") + + +def test_parse_info_from_known_good_empty_module_dict_in_json(tmp_path: Path): + """Test that assertion works if module dictionary is empty""" + json_file = tmp_path / "emample.json" + _ = json_file.write_text('{"another_key": {"a": "b"}, "modules": {}}') + + with pytest.raises( + AssertionError, + match=f"Known good json at: {json_file} has an empty 'modules' dictionary", + ): + parse_info_from_known_good(json_file, "any_module") + + +# Tests for get_github_link_from_json +def test_get_github_link_from_json_happy_path(): + """Test generating GitHub link from metadata.""" + metadata = ModuleInfo( + name="project_name", + url="https://github.com/eclipse/project", + hash="commit123abc", + ) + + link = DefaultNeedLink() + link.file = Path("docs/index.rst") + link.line = 100 + + result = get_github_link_from_json(metadata, link) + + assert ( + result + == "https://github.com/eclipse/project/blob/commit123abc/docs/index.rst#L100" + ) + + +def test_get_github_link_from_json_with_none_link(): + """Test that None link creates DefaultNeedLink.""" + metadata = ModuleInfo( + name="test_repo", url="https://github.com/test/repo", hash="def456" + ) + + result = get_github_link_from_json(metadata, None) + + assert result.startswith("https://github.com/test/repo/blob/def456/") + assert "#L" in result + + +def test_get_github_link_from_json_with_line_zero(): + """Test generating link with line number 0.""" + metadata = ModuleInfo( + name="test_repo", url="https://github.com/test/repo", hash="hash123" + ) + + link = DefaultNeedLink() + link.file = Path("file.py") + link.line = 0 + + result = get_github_link_from_json(metadata, link) + + assert result == "https://github.com/test/repo/blob/hash123/file.py#L0" + + +# Tests for get_github_link +def test_get_github_link_with_hash(): + """Test get_github_link uses json method when hash is present.""" + metadata = ModuleInfo( + name="some_module", url="https://github.com/org/repo", hash="commit_hash_123" + ) + + link = DefaultNeedLink() + link.file = Path("src/example.py") + link.line = 42 + + result = get_github_link(metadata, link) + + assert ( + result == "https://github.com/org/repo/blob/commit_hash_123/src/example.py#L42" + ) + +@pytest.fixture +def temp_dir() -> Generator[Path, None, None]: + """Create a temporary directory for tests.""" + with tempfile.TemporaryDirectory() as temp_dir: + yield Path(temp_dir) + +@pytest.fixture +def git_repo(temp_dir: Path) -> Path: + """Create a real git repository for testing.""" + git_dir: Path = temp_dir / "test_repo" + git_dir.mkdir() + + # Initialize git repo + _ = subprocess.run(["git", "init"], cwd=git_dir, check=True, capture_output=True) + _ = subprocess.run( + ["git", "config", "user.email", "test@example.com"], cwd=git_dir, check=True + ) + _ = subprocess.run( + ["git", "config", "user.name", "Test User"], cwd=git_dir, check=True + ) + + # Create a test file and commit + test_file: Path = git_dir / "test_file.py" + _ = test_file.write_text("# Test file\nprint('hello')\n") + _ = subprocess.run(["git", "add", "."], cwd=git_dir, check=True) + _ = subprocess.run( + ["git", "commit", "-m", "Initial commit"], cwd=git_dir, check=True + ) + + # Add a remote + _ = subprocess.run( + ["git", "remote", "add", "origin", "git@github.com:test-user/test-repo.git"], + cwd=git_dir, + check=True, + ) + return git_dir + + +def test_get_github_link_with_real_repo(git_repo: Path) -> None: + """ + Test generating GitHub link without url/hash. + This expects to be in a git module + """ + metadata = ModuleInfo(name="some_module", url="", hash="") + + link = DefaultNeedLink() + link.file = Path("src/example.py") + link.line = 42 + + # Have to change directories in order to ensure that we get the right/any .git file + os.chdir(Path(git_repo).absolute()) + # ADAPTED: Using get_github_link_from_git for direct local repo testing + result = get_github_link(metadata, link) + + # Should contain the base URL, hash, file path, and line number + assert "https://github.com/test-user/test-repo/blob/" in result + assert "src/example.py#L42" in result + + # Verify the hash is actually from the repo + hash_from_link = result.split("/blob/")[1].split("/")[0] + actual_hash = get_current_git_hash(git_repo) + assert hash_from_link == actual_hash + + +def test_complete_workflow(known_good_json: Path): + """Test complete workflow from path to GitHub link.""" + + # Parse path + path = Path("external/score_docs_as_code+/src/helper_lib/test_helper_lib.py") + module_name = parse_module_name_from_path(path) + assert module_name == "score_docs_as_code" + + # Get metadata + hash_val, repo_url = parse_info_from_known_good(known_good_json, module_name) + assert hash_val == "c1207676afe6cafd25c35d420e73279a799515d8" + assert repo_url == "https://github.com/eclipse-score/docs-as-code" + + # Generate link + metadata = ModuleInfo(name=module_name, url=repo_url, hash=hash_val) + link = DefaultNeedLink() + link.file = Path("src/helper_lib/test_helper_lib.py") + link.line = 75 + + result = get_github_link(metadata, link) + + assert ( + result + == "https://github.com/eclipse-score/docs-as-code/blob/c1207676afe6cafd25c35d420e73279a799515d8/src/helper_lib/test_helper_lib.py#L75" + ) diff --git a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py index fb4780dc1..e49d856da 100644 --- a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py +++ b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py @@ -26,6 +26,7 @@ from sphinx_needs.data import SphinxNeedsData from src.extensions.score_source_code_linker.needlinks import NeedLink +from src.extensions.score_source_code_linker.module_source_links import ModuleInfo from src.extensions.score_source_code_linker.testlink import ( DataForTestLink, DataForTestLink_JSON_Decoder, @@ -40,12 +41,12 @@ from src.extensions.score_source_code_linker.helpers import get_github_link -@pytest.fixture() +@pytest.fixture(scope="session") def sphinx_base_dir(tmp_path_factory: TempPathFactory) -> Path: return tmp_path_factory.mktemp("test_git_repo") -@pytest.fixture() +@pytest.fixture(scope="session") def git_repo_setup(sphinx_base_dir: Path) -> Path: """Creating git repo, to make testing possible""" @@ -67,7 +68,7 @@ def git_repo_setup(sphinx_base_dir: Path) -> Path: return repo_path -@pytest.fixture() +@pytest.fixture(scope="session") def create_demo_files(sphinx_base_dir: Path, git_repo_setup: Path): repo_path = sphinx_base_dir @@ -312,6 +313,9 @@ def example_source_link_text_all_ok(sphinx_base_dir: Path) -> dict[str, list[Nee tag="#" + " req-Id:", need="TREQ_ID_1", full_line="#" + " req-Id: TREQ_ID_1", + module_name="local_module", + url="", + hash="", ), NeedLink( file=Path("src/implementation1.py"), @@ -319,6 +323,9 @@ def example_source_link_text_all_ok(sphinx_base_dir: Path) -> dict[str, list[Nee tag="#" + " req-Id:", need="TREQ_ID_1", full_line="#" + " req-Id: TREQ_ID_1", + module_name="local_module", + url="", + hash="", ), ], "TREQ_ID_2": [ @@ -328,6 +335,9 @@ def example_source_link_text_all_ok(sphinx_base_dir: Path) -> dict[str, list[Nee tag="#" + " req-Id:", need="TREQ_ID_2", full_line="#" + " req-Id: TREQ_ID_2", + module_name="local_module", + url="", + hash="", ) ], "TREQ_ID_3": [], @@ -346,6 +356,9 @@ def example_test_link_text_all_ok(sphinx_base_dir: Path): verify_type="fully", result="passed", result_text="", + module_name="local_module", + url="", + hash="", ), ], "TREQ_ID_2": [ @@ -357,6 +370,9 @@ def example_test_link_text_all_ok(sphinx_base_dir: Path): verify_type="partially", result="passed", result_text="", + module_name="local_module", + url="", + hash="", ), DataForTestLink( name="test_error_handling", @@ -366,6 +382,9 @@ def example_test_link_text_all_ok(sphinx_base_dir: Path): verify_type="partially", result="passed", result_text="", + module_name="local_module", + url="", + hash="", ), ], "TREQ_ID_3": [ @@ -377,6 +396,9 @@ def example_test_link_text_all_ok(sphinx_base_dir: Path): verify_type="partially", result="passed", result_text="", + module_name="local_module", + url="", + hash="", ), DataForTestLink( name="test_error_handling", @@ -386,6 +408,9 @@ def example_test_link_text_all_ok(sphinx_base_dir: Path): verify_type="partially", result="passed", result_text="", + module_name="local_module", + url="", + hash="", ), ], } @@ -409,11 +434,21 @@ def example_source_link_text_non_existent(sphinx_base_dir: Path): def make_source_link(needlinks: list[NeedLink]): - return ", ".join(f"{get_github_link(n)}<>{n.file}:{n.line}" for n in needlinks) + metadata= ModuleInfo( + name="local_module", + url="", + hash="", + ) + return ", ".join(f"{get_github_link(metadata,n)}<>{n.file}:{n.line}" for n in needlinks) def make_test_link(testlinks: list[DataForTestLink]): - return ", ".join(f"{get_github_link(n)}<>{n.name}" for n in testlinks) + metadata= ModuleInfo( + name="local_module", + url="", + hash="", + ) + return ", ".join(f"{get_github_link(metadata,n)}<>{n.name}" for n in testlinks) def compare_json_files( From 0d1e490f205db22fac4d849fab2ff11055538af2 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Tue, 17 Mar 2026 10:22:12 +0100 Subject: [PATCH 16/34] Fixing tests --- .../module_source_links.py | 40 +- .../need_source_links.py | 6 +- .../score_source_code_linker/needlinks.py | 8 +- .../tests/expected_module_grouped.json | 83 +++ .../tests/test_codelink.py | 614 +++++++++++++++--- .../tests/test_module_source_links.py | 467 +++++++++++++ .../test_module_source_links_integration.py | 456 +++++++++++++ .../tests/test_testlink.py | 228 +++++++ 8 files changed, 1799 insertions(+), 103 deletions(-) create mode 100644 src/extensions/score_source_code_linker/tests/expected_module_grouped.json create mode 100644 src/extensions/score_source_code_linker/tests/test_module_source_links.py create mode 100644 src/extensions/score_source_code_linker/tests/test_module_source_links_integration.py diff --git a/src/extensions/score_source_code_linker/module_source_links.py b/src/extensions/score_source_code_linker/module_source_links.py index bf299a49f..cc4e39165 100644 --- a/src/extensions/score_source_code_linker/module_source_links.py +++ b/src/extensions/score_source_code_linker/module_source_links.py @@ -48,8 +48,25 @@ def default(self, o: object): # (hash, module_name, url) if isinstance(o, NeedLink | DataForTestLink): return o.to_dict_without_metadata() - if isinstance(o, ModuleSourceLinks | SourceCodeLinks | NeedSourceLinks): - return asdict(o) + # We need to split this up, otherwise the nested + # dictionaries won't get split up and we will not + # run into the 'to_dict_without_metadata' as + # everything will be converted to a normal dictionary + if isinstance(o, ModuleSourceLinks): + return { + "module": asdict(o.module), + "needs": o.needs, # Let the encoder handle the list + } + if isinstance(o, SourceCodeLinks): + return { + "need": o.need, + "links": o.links, + } + if isinstance(o, NeedSourceLinks): + return { + "CodeLinks": o.CodeLinks, + "TestLinks": o.TestLinks, + } return super().default(o) @@ -61,7 +78,7 @@ def ModuleSourceLinks_JSON_Decoder( needs = d["needs"] return ModuleSourceLinks( module=ModuleInfo( - name=module.get("module_name"), + name=module.get("name"), hash=module.get("hash"), url=module.get("url"), ), @@ -75,10 +92,10 @@ def ModuleSourceLinks_JSON_Decoder( def store_module_source_links_json( file: Path, source_code_links: list[ModuleSourceLinks] ): - # After `rm -rf _build` or on clean builds the directory does not exist, so we need - # to create it - file.parent.mkdir(exist_ok=True) - with open(file, "w", encoding="utf-8") as f: + # After `rm -rf _build` or on clean builds the directory does not exist, + # so we need to create it. We create any folder that might be missing + file.parent.mkdir(exist_ok=True, parents=True) + with open(file, "w") as f: json.dump( source_code_links, f, @@ -94,14 +111,11 @@ def load_module_source_links_json(file: Path) -> list[ModuleSourceLinks]: object_hook=ModuleSourceLinks_JSON_Decoder, ) assert isinstance(links, list), ( - "The ModuleSourceLink json should be aa list of ModuleSourceLink objects." + "The ModuleSourceLink json should be a list of ModuleSourceLink objects." ) - print("=====================") - print("=== TESTING LINKS IN ModuleSourceLink === ") for link in links: if not isinstance(link, ModuleSourceLinks): print(f"Link not module_sourcelink: {link}") - print("=====================") assert all(isinstance(link, ModuleSourceLinks) for link in links), ( "All items in module source link cache should be ModuleSourceLink objects." ) @@ -129,7 +143,9 @@ def group_needs_by_module(links: list[SourceCodeLinks]) -> list[ModuleSourceLink ) ) - module_groups[module_key].needs.append(source_link) # Much clearer! + # TODO: Add an assert that checks if needs only are + # in a singular module (not allowed to be in multiple) + module_groups[module_key].needs.append(source_link) return [ ModuleSourceLinks(module=group.module, needs=group.needs) diff --git a/src/extensions/score_source_code_linker/need_source_links.py b/src/extensions/score_source_code_linker/need_source_links.py index 1582c2fe8..a0b79d796 100644 --- a/src/extensions/score_source_code_linker/need_source_links.py +++ b/src/extensions/score_source_code_linker/need_source_links.py @@ -82,9 +82,9 @@ def SourceCodeLinks_JSON_Decoder(d: dict[str, Any]) -> SourceCodeLinks | dict[st def store_source_code_links_combined_json( file: Path, source_code_links: list[SourceCodeLinks] ): - # After `rm -rf _build` or on clean builds the directory does not exist, so we need - # to create it - file.parent.mkdir(exist_ok=True) + # After `rm -rf _build` or on clean builds the directory does not exist, + # so we need to create it. We create any folder that might be missing + file.parent.mkdir(exist_ok=True, parents=True) with open(file, "w") as f: json.dump( source_code_links, diff --git a/src/extensions/score_source_code_linker/needlinks.py b/src/extensions/score_source_code_linker/needlinks.py index 0f55d6dd3..588c95fef 100644 --- a/src/extensions/score_source_code_linker/needlinks.py +++ b/src/extensions/score_source_code_linker/needlinks.py @@ -140,7 +140,9 @@ def store_source_code_links_with_metadata_json( """ payload: list[object] = [metadata, *needlist] - file.parent.mkdir(exist_ok=True) + # After `rm -rf _build` or on clean builds the directory does not exist, + # so we need to create it. We create any folder that might be missing + file.parent.mkdir(exist_ok=True,parents=True) with open(file, "w", encoding="utf-8") as f: json.dump(payload, f, cls=NeedLinkEncoder, indent=2, ensure_ascii=False) @@ -151,7 +153,9 @@ def store_source_code_links_json(file: Path, needlist: list[NeedLink]) -> None: [ needlink1, needlink2, ... ] """ - file.parent.mkdir(exist_ok=True) + # After `rm -rf _build` or on clean builds the directory does not exist, + # so we need to create it. We create any folder that might be missing + file.parent.mkdir(exist_ok=True, parents=True) with open(file, "w", encoding="utf-8") as f: json.dump(needlist, f, cls=NeedLinkEncoder, indent=2, ensure_ascii=False) diff --git a/src/extensions/score_source_code_linker/tests/expected_module_grouped.json b/src/extensions/score_source_code_linker/tests/expected_module_grouped.json new file mode 100644 index 000000000..fd1ebfd79 --- /dev/null +++ b/src/extensions/score_source_code_linker/tests/expected_module_grouped.json @@ -0,0 +1,83 @@ +[ + { + "module": { + "name": "local_module", + "hash": "", + "url": "" + }, + "needs": [ + { + "need": "MOD_REQ_1", + "links": { + "CodeLinks": [ + { + "file": "src/module_a_impl.py", + "line": 3, + "tag": "# req-Id:", + "need": "MOD_REQ_1", + "full_line": "# req-Id: MOD_REQ_1" + }, + { + "file": "src/module_b_impl.py", + "line": 3, + "tag": "# req-Id:", + "need": "MOD_REQ_1", + "full_line": "# req-Id: MOD_REQ_1" + } + ], + "TestLinks": [ + { + "name": "test_module_a", + "file": "src/test_module_a.py", + "line": 10, + "need": "MOD_REQ_1", + "verify_type": "fully", + "result": "passed", + "result_text": "" + } + ] + } + }, + { + "need": "MOD_REQ_2", + "links": { + "CodeLinks": [ + { + "file": "src/module_a_impl.py", + "line": 7, + "tag": "# req-Id:", + "need": "MOD_REQ_2", + "full_line": "# req-Id: MOD_REQ_2" + } + ], + "TestLinks": [] + } + }, + { + "need": "MOD_REQ_3", + "links": { + "CodeLinks": [ + { + "file": "src/module_b_impl.py", + "line": 7, + "tag": "# req-Id:", + "need": "MOD_REQ_3", + "full_line": "# req-Id: MOD_REQ_3" + } + ], + "TestLinks": [ + { + "name": "test_module_b", + "file": "src/test_module_b.py", + "line": 20, + "need": "MOD_REQ_3", + "verify_type": "partially", + "result": "passed", + "result_text": "" + } + ] + } + } + ] + } +] diff --git a/src/extensions/score_source_code_linker/tests/test_codelink.py b/src/extensions/score_source_code_linker/tests/test_codelink.py index 46d45a59c..e65adcb08 100644 --- a/src/extensions/score_source_code_linker/tests/test_codelink.py +++ b/src/extensions/score_source_code_linker/tests/test_codelink.py @@ -10,8 +10,14 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* + +# ╓ ╖ +# ║ Some portions created by Co-Pilot ║ +# ╙ ╜ + import json import os +from pkgutil import ModuleInfo import subprocess import tempfile from collections.abc import Generator @@ -20,7 +26,11 @@ from typing import Any import pytest + +# S-CORE plugin to allow for properties/attributes in xml +# Enables Testlinking from attribute_plugin import add_test_properties # type: ignore[import-untyped] + from sphinx_needs.data import NeedsInfoType, NeedsMutable from sphinx_needs.need_item import ( NeedItem, @@ -28,39 +38,35 @@ NeedsContent, ) -# Import the module under test -# Note: You'll need to adjust these imports based on your actual module structure from src.extensions.score_source_code_linker import ( find_need, get_cache_filename, group_by_need, ) + +from src.extensions.score_source_code_linker.module_source_links import ModuleInfo + from src.extensions.score_source_code_linker.needlinks import ( + DefaultNeedLink, + MetaData, NeedLink, + NeedLinkEncoder, + is_metadata, load_source_code_links_json, + load_source_code_links_with_metadata_json, + needlink_decoder, store_source_code_links_json, + store_source_code_links_with_metadata_json, ) from src.helper_lib import ( get_current_git_hash, ) -# ADAPTED: Importing from the new generator location + from src.extensions.score_source_code_linker.helpers import ( get_github_link_from_git, + get_github_link, ) -""" -# ────────────────ATTENTION─────────────── - -# ╭──────────────────────────────────────╮ -# │ !!!!! │ -# │ BOILERPLATE TEST MADE VIA │ -# │ GENERATION. NOT YET FULLY LOOKED │ -# │ THROUGH │ -# │ !!!! │ -# ╰──────────────────────────────────────╯ - -""" - def test_need(**kwargs: Any) -> NeedItem: """Convenience function to create a NeedItem object with some defaults.""" @@ -130,8 +136,19 @@ def default(self, o: object): def needlink_test_decoder(d: dict[str, Any]) -> NeedLink | dict[str, Any]: - # ADAPTED: Updated to include new optional fields (module_name, hash, url) - if {"file", "line", "tag", "need", "full_line", "module_name", "hash" ,"url" } <= d.keys(): + """ + Since we have our own decoder, we have to ensure it works as expected + """ + if { + "file", + "line", + "tag", + "need", + "full_line", + "module_name", + "hash", + "url", + } <= d.keys(): return NeedLink( file=Path(d["file"]), line=d["line"], @@ -191,6 +208,9 @@ def sample_needlinks() -> list[NeedLink]: tag="#" + " req-Id:", need="TREQ_ID_1", full_line="#" + " req-Id: TREQ_ID_1", + module_name="local_module", + hash="", + url="", ), NeedLink( file=Path("src/implementation2.py"), @@ -198,6 +218,9 @@ def sample_needlinks() -> list[NeedLink]: tag="#" + " req-Id:", need="TREQ_ID_1", full_line="#" + " req-Id: TREQ_ID_1", + module_name="local_module", + hash="", + url="", ), NeedLink( file=Path("src/implementation1.py"), @@ -205,6 +228,9 @@ def sample_needlinks() -> list[NeedLink]: tag="#" + " req-Id:", need="TREQ_ID_2", full_line="#" + " req-Id: TREQ_ID_2", + module_name="local_module", + hash="", + url="", ), NeedLink( file=Path("src/bad_implementation.py"), @@ -212,6 +238,9 @@ def sample_needlinks() -> list[NeedLink]: tag="#" + " req-Id:", need="TREQ_ID_200", full_line="#" + " req-Id: TREQ_ID_200", + module_name="local_module", + hash="", + url="", ), ] @@ -246,12 +275,6 @@ def sample_needs() -> dict[str, dict[str, str]]: } -# Test utility functions -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_get_cache_filename(): """Test cache filename generation.""" build_dir = Path("/tmp/build") @@ -260,17 +283,13 @@ def test_get_cache_filename(): assert result == expected +# Done to appease the LSP Gods def make_needs(needs_dict: dict[str, dict[str, Any]]) -> NeedsMutable: return NeedsMutable( {need_id: test_need(**params) for need_id, params in needs_dict.items()} ) -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_find_need_direct_match(): """Test finding a need with direct ID match.""" all_needs = make_needs( @@ -284,11 +303,6 @@ def test_find_need_direct_match(): assert result["id"] == "REQ_001" -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_find_need_not_found(): """Test finding a need that doesn't exist.""" all_needs = make_needs( @@ -301,11 +315,6 @@ def test_find_need_not_found(): assert result is None -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_group_by_need(sample_needlinks: list[NeedLink]) -> None: """Test grouping source code links by need ID.""" result = group_by_need(sample_needlinks) @@ -325,25 +334,12 @@ def test_group_by_need(sample_needlinks: list[NeedLink]) -> None: assert len(found_link.links.CodeLinks) == 1 -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_group_by_need_empty_list(): """Test grouping empty list of needlinks.""" result = group_by_need([], []) assert len(result) == 0 - - -# Test cache file operations -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_cache_file_operations( temp_dir: Path, sample_needlinks: list[NeedLink] ) -> None: @@ -370,11 +366,6 @@ def test_cache_file_operations( assert loaded_links[3].line == 2 -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_cache_file_with_encoded_comments(temp_dir: Path) -> None: """Test that cache file properly handles encoded comments.""" # Create needlinks with spaces in tags and full_line @@ -385,6 +376,9 @@ def test_cache_file_with_encoded_comments(temp_dir: Path) -> None: tag="#" + " req-Id:", need="TEST_001", full_line="#" + " req-Id: TEST_001", + module_name="local_module", + hash="", + url="", ) ] @@ -404,14 +398,6 @@ def test_cache_file_with_encoded_comments(temp_dir: Path) -> None: assert loaded_links[0].full_line == "#" + " req-Id: TEST_001" -# Integration tests - - -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_group_by_need_and_find_need_integration( sample_needlinks: list[NeedLink], ) -> None: @@ -435,11 +421,6 @@ def test_group_by_need_and_find_need_integration( assert found_need["id"] == found_link.need -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_source_linker_end_to_end_with_real_files( temp_dir: Path, git_repo: Path ) -> None: @@ -448,7 +429,7 @@ def test_source_linker_end_to_end_with_real_files( src_dir: Path = git_repo / "src" src_dir.mkdir() - (src_dir / "implementation1.py").write_text( + _ = (src_dir / "implementation1.py").write_text( """ # Some implementation #""" @@ -464,7 +445,7 @@ def function2(): """ ) - (src_dir / "implementation2.py").write_text( + _ = (src_dir / "implementation2.py").write_text( """ # Another implementation #""" @@ -475,8 +456,8 @@ def another_function(): ) # Commit the changes - subprocess.run(["git", "add", "."], cwd=git_repo, check=True) - subprocess.run( + _ = subprocess.run(["git", "add", "."], cwd=git_repo, check=True) + _ = subprocess.run( ["git", "commit", "-m", "Add implementation files"], cwd=git_repo, check=True ) @@ -489,6 +470,9 @@ def another_function(): tag="#" + " req-Id:", need="TREQ_ID_1", full_line="#" + " req-Id: TREQ_ID_1", + module_name="local_module", + hash="", + url="", ), NeedLink( file=Path("src/implementation1.py"), @@ -496,6 +480,9 @@ def another_function(): tag="#" + " req-Id:", need="TREQ_ID_2", full_line="#" + " req-Id: TREQ_ID_2", + module_name="local_module", + hash="", + url="", ), NeedLink( file=Path("src/implementation2.py"), @@ -503,6 +490,9 @@ def another_function(): tag="#" + " req-Id:", need="TREQ_ID_1", full_line="#" + " req-Id: TREQ_ID_1", + module_name="local_module", + hash="", + url="", ), ] @@ -526,18 +516,13 @@ def another_function(): # Test GitHub link generation # Have to change directories in order to ensure that we get the right/any .git file os.chdir(Path(git_repo).absolute()) + metadata = ModuleInfo(name="local_module", hash="", url="") for needlink in loaded_links: - # ADAPTED: Using get_github_link_from_git - github_link = get_github_link_from_git(needlink) + github_link = get_github_link(metadata, needlink) assert "https://github.com/test-user/test-repo/blob/" in github_link assert f"src/{needlink.file.name}#L{needlink.line}" in github_link -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_multiple_commits_hash_consistency(git_repo: Path) -> None: """Test that git hash remains consistent and links update properly.""" # Get initial hash @@ -545,9 +530,11 @@ def test_multiple_commits_hash_consistency(git_repo: Path) -> None: # Create and commit a new file new_file: Path = git_repo / "new_file.py" - new_file.write_text("# New file\nprint('new')") - subprocess.run(["git", "add", "."], cwd=git_repo, check=True) - subprocess.run(["git", "commit", "-m", "Add new file"], cwd=git_repo, check=True) + _ = new_file.write_text("# New file\nprint('new')") + _ = subprocess.run(["git", "add", "."], cwd=git_repo, check=True) + _ = subprocess.run( + ["git", "commit", "-m", "Add new file"], cwd=git_repo, check=True + ) # Get new hash new_hash = get_current_git_hash(git_repo) @@ -565,7 +552,462 @@ def test_multiple_commits_hash_consistency(git_repo: Path) -> None: full_line="#" + " req-Id: TREQ_ID_1", ) + metadata = ModuleInfo(name="local_module", hash="", url="") os.chdir(Path(git_repo).absolute()) - # ADAPTED: Using get_github_link_from_git - github_link = get_github_link_from_git(needlink) + github_link = get_github_link(metadata, needlink) assert new_hash in github_link + + +def test_is_metadata_missing_keys(): + """Bad path: Dict missing required keys returns False""" + incomplete = {"module_name": "test", "hash": "abc"} + assert is_metadata(incomplete) is False + + +# ─────────────────[ NeedLink Dataclass Tests ]───────────────── + + +def test_needlink_to_dict_without_metadata(): + """to_dict_without_metadata should return NeedLink without metadata""" + needlink = NeedLink( + file=Path("src/test.py"), + line=10, + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + module_name="test_module", + url="https://github.com/test/repo", + hash="abc123", + ) + result = needlink.to_dict_without_metadata() + + assert "module_name" not in result + assert "hash" not in result + assert "url" not in result + assert result["need"] == "REQ_1" + assert result["line"] == 10 + + +def test_needlink_to_dict_full(): + """to_dict_full includes all fields""" + needlink = NeedLink( + file=Path("src/test.py"), + line=10, + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + module_name="test_module", + url="https://github.com/test/repo", + hash="abc123", + ) + result = needlink.to_dict_full() + + assert result["module_name"] == "test_module" + assert result["hash"] == "abc123" + assert result["url"] == "https://github.com/test/repo" + assert result["need"] == "REQ_1" + + +# ────────────────────[ JSON Encoder Tests ]──────────────────── + + +def test_needlink_encoder_includes_metadata(): + """Encoder includes all fields""" + encoder = NeedLinkEncoder() + needlink = NeedLink( + file=Path("src/test.py"), + line=10, + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + module_name="test_module", + url="https://github.com/test/repo", + hash="abc123", + ) + result = encoder.default(needlink) + + assert result["module_name"] == "test_module" + assert result["hash"] == "abc123" + assert result["url"] == "https://github.com/test/repo" + + +# ============================================================================ +# JSON Decoder Tests +# ============================================================================ + + +def test_needlink_decoder_with_all_fields(): + """Decoder reconstructs NeedLink with all fields""" + json_data = { + "file": "src/test.py", + "line": 10, + "tag": "# req-Id:", + "need": "REQ_1", + "full_line": "# req-Id: REQ_1", + "module_name": "test_module", + "hash": "abc123", + "url": "https://github.com/test/repo", + } + result = needlink_decoder(json_data) + + assert isinstance(result, NeedLink) + assert result.file == Path("src/test.py") + assert result.line == 10 + assert result.module_name == "test_module" + + +def test_needlink_decoder_non_needlink_dict(): + """Edge case: Non-NeedLink dict is returned unchanged""" + json_data = {"some": "other", "data": "structure"} + result = needlink_decoder(json_data) + assert result == json_data + + +# ───────────────[ Testing Encoding / Decoding ]───────────── + + +def test_store_and_load_source_code_links(tmp_path: Path): + """Happy path: Store and load without metadata""" + needlinks = [ + NeedLink( + file=Path("src/impl.py"), + line=42, + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + module_name="mod", + url="url", + hash="hash", + ) + ] + + test_file = tmp_path / "standard.json" + store_source_code_links_json(test_file, needlinks) + loaded = load_source_code_links_json(test_file) + + assert len(loaded) == 1 + assert loaded[0].need == "REQ_1" + assert loaded[0].module_name == "mod" + + +def test_load_validates_list_type(tmp_path: Path): + """Bad path: Non-list JSON fails validation""" + test_file = tmp_path / "not_list.json" + _ = test_file.write_text('{"file": "src/test.py", "line": 1}') + + with pytest.raises(AssertionError, match="should be a list"): + load_source_code_links_json(test_file) + + +def test_load_validates_all_items_are_needlinks(tmp_path: Path): + """Bad path: List with invalid items fails validation""" + test_file = tmp_path / "invalid_items.json" + _ = test_file.write_text('[{"invalid": "item"}]') + + with pytest.raises(AssertionError, match="should be NeedLink objects"): + load_source_code_links_json(test_file) + + +# Adding additional tests to test the metadata stuff (excluding it if wanted) + + +def test_store_and_load_with_metadata(tmp_path: Path): + """Happy path: Store with metadata dict and load correctly""" + metadata: MetaData = { + "module_name": "external_module", + "hash": "commit_xyz", + "url": "https://github.com/external/repo", + } + needlinks = [ + NeedLink( + file=Path("src/impl.py"), + line=10, + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + module_name="", # Will be filled from metadata + url="", + hash="", + ), + NeedLink( + file=Path("src/impl2.py"), + line=20, + tag="# req-Id:", + need="REQ_2", + full_line="# req-Id: REQ_2", + module_name="", + url="", + hash="", + ), + ] + + test_file = tmp_path / "with_metadata.json" + store_source_code_links_with_metadata_json(test_file, metadata, needlinks) + loaded = load_source_code_links_with_metadata_json(test_file) + + assert len(loaded) == 2 + # Verify metadata was applied to all links + assert loaded[0].module_name == "external_module" + assert loaded[0].hash == "commit_xyz" + assert loaded[0].url == "https://github.com/external/repo" + assert loaded[1].module_name == "external_module" + + +def test_load_with_metadata_missing_metadata_dict(tmp_path: Path): + """Bad path: Loading file without metadata dict raises TypeError""" + # Store without metadata (just needlinks) + needlinks = [ + NeedLink( + file=Path("src/test.py"), + line=1, + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + ) + ] + + test_file = tmp_path / "no_metadata.json" + store_source_code_links_json(test_file, needlinks) + + # Try to load as if it has metadata + with pytest.raises(TypeError, match="you might wanted to call"): + load_source_code_links_with_metadata_json(test_file) + + +def test_load_with_metadata_invalid_items_after_metadata(tmp_path: Path): + """Bad path: Items after metadata dict are not NeedLinks""" + test_file = tmp_path / "bad_items.json" + # Manually create invalid JSON + _ = test_file.write_text( + json.dumps( + [{"module_name": "mod", "hash": "h", "url": "u"}, {"invalid": "structure"}] + ) + ) + + with pytest.raises(TypeError, match="must decode to NeedLink objects"): + load_source_code_links_with_metadata_json(test_file) + + +# ────────────────[ File Path Resolution Tests ]──────────────── + + +def test_load_resolves_relative_path_with_env_var(tmp_path: Path, monkeypatch): + """Edge case: Relative path is resolved using BUILD_WORKSPACE_DIRECTORY""" + workspace = tmp_path / "workspace" + workspace.mkdir() + + needlinks = [ + NeedLink( + file=Path("src/test.py"), + line=1, + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + ) + ] + + # Store in workspace + cache_file = workspace / "cache.json" + store_source_code_links_json(cache_file, needlinks) + + # Set env var and load with relative path + monkeypatch.setenv("BUILD_WORKSPACE_DIRECTORY", str(workspace)) + loaded = load_source_code_links_json(Path("cache.json")) + + assert len(loaded) == 1 + assert loaded[0].need == "REQ_1" + + +def test_load_with_metadata_resolves_relative_path(tmp_path: Path, monkeypatch): + """Edge case: load_with_metadata resolves relative paths using env var""" + workspace = tmp_path / "workspace" + workspace.mkdir() + + metadata: MetaData = { + "module_name": "mod", + "hash": "h", + "url": "u", + } + needlinks = [ + NeedLink( + file=Path("src/test.py"), + line=1, + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + ) + ] + + cache_file = workspace / "metadata_cache.json" + store_source_code_links_with_metadata_json(cache_file, metadata, needlinks) + + monkeypatch.setenv("BUILD_WORKSPACE_DIRECTORY", str(workspace)) + loaded = load_source_code_links_with_metadata_json(Path("metadata_cache.json")) + + assert len(loaded) == 1 + assert loaded[0].module_name == "mod" + + +# ─────────────────────[ Roundtrip Tests ]─────────────────── + + +def test_roundtrip_standard_format(tmp_path: Path): + """Happy path: Standard format preserves all data""" + needlinks = [ + NeedLink( + file=Path("src/file1.py"), + line=10, + tag="# req-Id:", + need="REQ_A", + full_line="# req-Id: REQ_A", + module_name="mod_a", + url="url_a", + hash="hash_a", + ), + NeedLink( + file=Path("src/file2.py"), + line=20, + tag="# req-Id:", + need="REQ_B", + full_line="# req-Id: REQ_B", + module_name="mod_b", + url="url_b", + hash="hash_b", + ), + ] + + test_file = tmp_path / "standard.json" + store_source_code_links_json(test_file, needlinks) + loaded = load_source_code_links_json(test_file) + + assert len(loaded) == 2 + assert loaded[0].module_name == "mod_a" + assert loaded[1].module_name == "mod_b" + + +def test_roundtrip_metadata_format_applies_metadata(tmp_path: Path): + """Happy path: Metadata format applies metadata to all links""" + metadata: MetaData = { + "module_name": "shared_module", + "hash": "shared_hash", + "url": "https://github.com/shared/repo", + } + needlinks = [ + NeedLink( + file=Path("src/f1.py"), + line=5, + tag="# req-Id:", + need="R1", + full_line="# req-Id: R1", + ), + NeedLink( + file=Path("src/f2.py"), + line=15, + tag="# req-Id:", + need="R2", + full_line="# req-Id: R2", + ), + ] + + test_file = tmp_path / "with_metadata.json" + store_source_code_links_with_metadata_json(test_file, metadata, needlinks) + loaded = load_source_code_links_with_metadata_json(test_file) + + assert len(loaded) == 2 + # Both should have the same metadata applied + for link in loaded: + assert link.module_name == "shared_module" + assert link.hash == "shared_hash" + assert link.url == "https://github.com/shared/repo" + + +def test_roundtrip_empty_lists(tmp_path: Path): + """Edge case: Empty list can be stored and loaded""" + test_file = tmp_path / "empty.json" + store_source_code_links_json(test_file, []) + loaded = load_source_code_links_json(test_file) + + assert loaded == [] + + +# ──────────────[ JSON Format Verification Tests ]────────────── + + +def test_json_format_with_metadata_has_separate_dict(tmp_path: Path): + """Edge case: Verify metadata format has metadata as first element""" + metadata: MetaData = { + "module_name": "test_mod", + "hash": "test_hash", + "url": "test_url", + } + needlink = NeedLink( + file=Path("src/test.py"), + line=1, + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + ) + + test_file = tmp_path / "metadata_format.json" + store_source_code_links_with_metadata_json(test_file, metadata, [needlink]) + + with open(test_file) as f: + raw_json = json.load(f) + + assert isinstance(raw_json, list) + assert len(raw_json) == 2 # metadata + 1 needlink + # First element should be metadata dict + assert raw_json[0]["module_name"] == "test_mod" + assert raw_json[0]["hash"] == "test_hash" + assert raw_json[0]["url"] == "test_url" + + +# ───────────[ NeedLink Equality and Hashing Tests ]───────── + + +def test_needlink_equality_same_values(): + """Happy path: Two NeedLinks with same values are equal""" + link1 = NeedLink( + file=Path("src/test.py"), + line=10, + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + module_name="mod", + url="url", + hash="hash", + ) + link2 = NeedLink( + file=Path("src/test.py"), + line=10, + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + module_name="mod", + url="url", + hash="hash", + ) + + assert link1 == link2 + assert hash(link1) == hash(link2) + + +def test_needlink_inequality_different_values(): + """Edge case: NeedLinks with different values are not equal""" + link1 = NeedLink( + file=Path("src/test.py"), + line=10, + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + ) + link2 = NeedLink( + file=Path("src/test.py"), + line=20, # Different line + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + ) + + assert link1 != link2 + assert hash(link1) != hash(link2) diff --git a/src/extensions/score_source_code_linker/tests/test_module_source_links.py b/src/extensions/score_source_code_linker/tests/test_module_source_links.py new file mode 100644 index 000000000..5119c3f0e --- /dev/null +++ b/src/extensions/score_source_code_linker/tests/test_module_source_links.py @@ -0,0 +1,467 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import json +from pathlib import Path + +import pytest + +from src.extensions.score_source_code_linker.module_source_links import ( + ModuleInfo, + ModuleSourceLinks, + ModuleSourceLinks_JSON_Decoder, + ModuleSourceLinks_JSON_Encoder, + group_needs_by_module, + load_module_source_links_json, + store_module_source_links_json, +) +from src.extensions.score_source_code_linker.need_source_links import ( + NeedSourceLinks, + SourceCodeLinks, +) +from src.extensions.score_source_code_linker.needlinks import NeedLink +from src.extensions.score_source_code_linker.testlink import DataForTestLink + + +""" + ────────────────INFORMATION─────────────── + +# ╭──────────────────────────────────────────────────────────╮ +# │ partially generated │ +# │ Human screened and adapted │ +# │ though still be aware of mistakes │ +# ╰──────────────────────────────────────────────────────────╯ + +""" + + +# ╭──────────────────────────────────────────────────────────╮ +# │ JSON ENCODER TEST │ +# ╰──────────────────────────────────────────────────────────╯ + + +def test_json_encoder_removes_metadata_from_needlink(): + """Happy path: NeedLink metadata fields are excluded from JSON output""" + encoder = ModuleSourceLinks_JSON_Encoder() + needlink = NeedLink( + file=Path("src/test.py"), + line=10, + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + module_name="test_module", + url="https://github.com/test/repo", + hash="abc123", + ) + result = encoder.default(needlink) + + assert "module_name" not in result + assert "url" not in result + assert "hash" not in result + assert result["need"] == "REQ_1" + assert result["line"] == 10 + + +def test_json_encoder_removes_metadata_from_testlink(): + """Happy path: DataForTestLink metadata fields are excluded from JSON output""" + encoder = ModuleSourceLinks_JSON_Encoder() + testlink = DataForTestLink( + name="test_something", + file=Path("src/test_file.py"), + need="REQ_1", + line=20, + verify_type="fully", + result="passed", + result_text="", + module_name="test_module", + url="https://github.com/test/repo", + hash="abc123", + ) + result = encoder.default(testlink) + + assert "module_name" not in result + assert "url" not in result + assert "hash" not in result + assert result["name"] == "test_something" + assert result["need"] == "REQ_1" + + +def test_json_encoder_converts_path_to_string(): + """Happy path: Path objects are converted to strings""" + encoder = ModuleSourceLinks_JSON_Encoder() + result = encoder.default(Path("/test/path/file.py")) + assert result == "/test/path/file.py" + assert isinstance(result, str) + + +# ============================================================================ +# JSON Decoder Tests +# ============================================================================ + + +def test_json_decoder_reconstructs_module_source_links(): + """Happy path: Valid JSON dict is decoded into ModuleSourceLinks""" + json_data = { + "module": {"name": "test_module", "hash": "hash1", "url": "url1"}, + "needs": [ + { + "need": "REQ_1", + "links": {"CodeLinks": [], "TestLinks": []}, + } + ], + } + result = ModuleSourceLinks_JSON_Decoder(json_data) + + assert isinstance(result, ModuleSourceLinks) + assert result.module.name == "test_module" + assert result.module.hash == "hash1" + assert result.module.url == "url1" + assert len(result.needs) == 1 + + +def test_json_decoder_returns_unchanged_for_non_module_dict(): + """Edge case: Non-ModuleSourceLinks dicts are returned unchanged""" + json_data = {"some": "data", "other": "values"} + result = ModuleSourceLinks_JSON_Decoder(json_data) + assert result == json_data + + +# ============================================================================ +# Store/Load Tests +# ============================================================================ + + +def test_store_and_load_roundtrip(tmp_path: Path): + """Happy path: Store and load preserves data correctly""" + module = ModuleInfo(name="test_module", hash="hash1", url="url1") + needlink = NeedLink( + file=Path("src/test.py"), + line=10, + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + module_name="test_module", + url="url1", + hash="hash1", + ) + scl = SourceCodeLinks( + need="REQ_1", + links=NeedSourceLinks(CodeLinks=[needlink], TestLinks=[]), + ) + module_links = ModuleSourceLinks(module=module, needs=[scl]) + + test_file = tmp_path / "test.json" + store_module_source_links_json(test_file, [module_links]) + loaded = load_module_source_links_json(test_file) + + assert len(loaded) == 1 + assert loaded[0].module.name == "test_module" + assert loaded[0].needs[0].need == "REQ_1" + + +def test_store_creates_parent_directories(tmp_path: Path): + """Edge case: Parent directories are created if they don't exist""" + nested_path = tmp_path / "nested" / "deeply" / "test.json" + module = ModuleInfo(name="test", hash="h", url="u") + module_links = ModuleSourceLinks(module=module, needs=[]) + + store_module_source_links_json(nested_path, [module_links]) + + assert nested_path.exists() + assert nested_path.parent.exists() + + +def test_load_empty_list(tmp_path: Path): + """Edge case: Loading empty list returns empty list""" + test_file = tmp_path / "empty.json" + store_module_source_links_json(test_file, []) + + loaded = load_module_source_links_json(test_file) + assert loaded == [] + + +def test_load_validates_is_list(tmp_path: Path): + """Bad path: Loading non-list JSON fails validation""" + test_file = tmp_path / "not_list.json" + test_file.write_text('{"module": {}, "needs": []}') + + with pytest.raises(AssertionError, match="should be a list"): + load_module_source_links_json(test_file) + + +def test_load_validates_items_are_correct_type(tmp_path: Path): + """Bad path: Loading list with invalid items fails validation""" + test_file = tmp_path / "invalid_items.json" + test_file.write_text('[{"invalid": "structure"}]') + + with pytest.raises(AssertionError, match="should be ModuleSourceLink objects"): + load_module_source_links_json(test_file) + + +# ============================================================================ +# group_needs_by_module Tests +# ============================================================================ + + +def test_group_needs_single_module_with_codelinks(): + """Happy path: Multiple needs from same module are grouped together""" + needlink1 = NeedLink( + file=Path("src/file1.py"), + line=10, + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + module_name="shared_module", + url="https://github.com/test/repo", + hash="hash1", + ) + needlink2 = NeedLink( + file=Path("src/file2.py"), + line=20, + tag="# req-Id:", + need="REQ_2", + full_line="# req-Id: REQ_2", + module_name="shared_module", + url="https://github.com/test/repo", + hash="hash1", + ) + + scl1 = SourceCodeLinks( + need="REQ_1", links=NeedSourceLinks(CodeLinks=[needlink1], TestLinks=[]) + ) + scl2 = SourceCodeLinks( + need="REQ_2", links=NeedSourceLinks(CodeLinks=[needlink2], TestLinks=[]) + ) + + result = group_needs_by_module([scl1, scl2]) + + assert len(result) == 1 + assert result[0].module.name == "shared_module" + assert result[0].module.hash == "hash1" + assert len(result[0].needs) == 2 + assert len(result[0].needs[0].links.CodeLinks) == 1 + assert len(result[0].needs[1].links.CodeLinks) == 1 + + +def test_group_needs_multiple_modules(): + """Happy path: Needs from different modules create separate groups""" + needlink_a = NeedLink( + file=Path("src/a.py"), + line=10, + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + module_name="module_a", + url="https://github.com/a/repo", + hash="hash_a", + ) + needlink_b = NeedLink( + file=Path("src/b.py"), + line=20, + tag="# req-Id:", + need="REQ_2", + full_line="# req-Id: REQ_2", + module_name="module_b", + url="https://github.com/b/repo", + hash="hash_b", + ) + + scl1 = SourceCodeLinks( + need="REQ_1", links=NeedSourceLinks(CodeLinks=[needlink_a], TestLinks=[]) + ) + scl2 = SourceCodeLinks( + need="REQ_2", links=NeedSourceLinks(CodeLinks=[needlink_b], TestLinks=[]) + ) + + result = group_needs_by_module([scl1, scl2]) + + assert len(result) == 2 + assert result[0].module.name == "module_a" + assert result[0].module.hash == "hash_a" + assert result[0].module.url == "https://github.com/a/repo" + assert result[1].module.name == "module_b" + assert result[1].module.hash == "hash_b" + assert result[1].module.url == "https://github.com/b/repo" + + +def test_group_needs_with_testlinks_only(): + """Happy path: Needs with only test links (no code links) are grouped correctly""" + testlink = DataForTestLink( + name="test_feature", + file=Path("tests/test.py"), + need="REQ_1", + line=15, + verify_type="fully", + result="passed", + result_text="", + module_name="test_module", + url="https://github.com/test/repo", + hash="hash1", + ) + testlink2 = DataForTestLink( + name="test_feature", + file=Path("tests/test2.py"), + need="REQ_10", + line=153, + verify_type="partially", + result="passed", + result_text="", + module_name="test_module", + url="https://github.com/test/repo", + hash="hash1", + ) + + scl = SourceCodeLinks( + need="REQ_1", + links=NeedSourceLinks(CodeLinks=[], TestLinks=[testlink]), + ) + scl2 = SourceCodeLinks( + need="REQ_10", + links=NeedSourceLinks(CodeLinks=[], TestLinks=[testlink2]), + ) + + result = group_needs_by_module([scl, scl2]) + + assert len(result) == 1 + assert result[0].module.name == "test_module" + assert len(result[0].needs) == 2 + needs = [x.need for x in result[0].needs] + assert "REQ_1" in needs + assert "REQ_10" in needs + assert len(result[0].needs[0].links.TestLinks) == 1 + assert len(result[0].needs[1].links.TestLinks) == 1 + + +def test_group_needs_with_testlinks_different_modules(): + """Testing Testlinks grouping for different modules""" + testlink = DataForTestLink( + name="test_feature", + file=Path("tests/test.py"), + need="REQ_1", + line=15, + verify_type="fully", + result="passed", + result_text="", + module_name="module_a", + url="https://github.com/test_a/repo_a", + hash="hash_a", + ) + testlink2 = DataForTestLink( + name="test_feature", + file=Path("tests/test3.py"), + need="REQ_10", + line=153, + verify_type="partially", + result="passed", + result_text="", + module_name="module_b", + url="https://github.com/test_b/repo_b", + hash="hash_b", + ) + + scl = SourceCodeLinks( + need="REQ_1", + links=NeedSourceLinks(CodeLinks=[], TestLinks=[testlink]), + ) + scl2 = SourceCodeLinks( + need="REQ_10", + links=NeedSourceLinks(CodeLinks=[], TestLinks=[testlink2]), + ) + + result = group_needs_by_module([scl, scl2]) + + assert len(result) == 2 + assert result[0].module.name == "module_a" + assert result[0].module.hash == "hash_a" + assert result[0].module.url == "https://github.com/test_a/repo_a" + assert result[1].module.name == "module_b" + assert result[1].module.hash == "hash_b" + assert result[1].module.url == "https://github.com/test_b/repo_b" + + +def test_group_needs_empty_list(): + """Edge case: Empty list returns empty result""" + result = group_needs_by_module([]) + assert result == [] + + +def test_group_needs_skips_needs_without_links(): + """ + Edge case: Needs with no code or test links are skipped + This should normally never even be possible to get + to this function. Though we should test what happens + """ + scl_with_links = SourceCodeLinks( + need="REQ_1", + links=NeedSourceLinks( + CodeLinks=[ + NeedLink( + file=Path("src/test.py"), + line=10, + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + module_name="module_a", + url="url1", + hash="hash1", + ) + ], + TestLinks=[], + ), + ) + scl_without_links = SourceCodeLinks( + need="REQ_2", + links=NeedSourceLinks(CodeLinks=[], TestLinks=[]), + ) + + result = group_needs_by_module([scl_with_links, scl_without_links]) + + assert len(result) == 1 + assert result[0].needs[0].need == "REQ_1" + assert result[0].needs[0].links.CodeLinks[0].full_line == "# req-Id: REQ_1" + + +def test_group_needs_mixed_codelinks_and_testlinks(): + """Happy path: Needs with both code and test links are handled correctly""" + needlink = NeedLink( + file=Path("src/impl.py"), + line=5, + tag="# req-Id:", + need="REQ_1", + full_line="# req-Id: REQ_1", + module_name="module_a", + url="https://github.com/test/repo", + hash="hash1", + ) + testlink = DataForTestLink( + name="test_impl", + file=Path("tests/test_impl.py"), + need="REQ_1", + line=10, + verify_type="fully", + result="passed", + result_text="", + module_name="module_a", + url="https://github.com/test/repo", + hash="hash1", + ) + + scl = SourceCodeLinks( + need="REQ_1", + links=NeedSourceLinks(CodeLinks=[needlink], TestLinks=[testlink]), + ) + + result = group_needs_by_module([scl]) + + assert len(result) == 1 + assert len(result[0].needs[0].links.CodeLinks) == 1 + assert len(result[0].needs[0].links.TestLinks) == 1 diff --git a/src/extensions/score_source_code_linker/tests/test_module_source_links_integration.py b/src/extensions/score_source_code_linker/tests/test_module_source_links_integration.py new file mode 100644 index 000000000..60c217c25 --- /dev/null +++ b/src/extensions/score_source_code_linker/tests/test_module_source_links_integration.py @@ -0,0 +1,456 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import contextlib +import json +import os +import shutil +import subprocess +from collections.abc import Callable +from pathlib import Path + +import pytest +from pytest import TempPathFactory +from sphinx.testing.util import SphinxTestApp + +from src.extensions.score_source_code_linker.module_source_links import ( + ModuleInfo, + ModuleSourceLinks_JSON_Decoder, + load_module_source_links_json, +) +from src.helper_lib import find_ws_root + + +""" + ────────────────INFORMATION─────────────── + +# ╭──────────────────────────────────────────────────────────╮ +# │ boiler plate generated │ +# │ Human screened and adapted │ +# │ though still be aware of mistakes │ +# ╰──────────────────────────────────────────────────────────╯ + +""" + + +@pytest.fixture(scope="session") +def sphinx_base_dir(tmp_path_factory: TempPathFactory) -> Path: + return tmp_path_factory.mktemp("test_module_links_repo") + + +@pytest.fixture(scope="session") +def git_repo_setup(sphinx_base_dir: Path) -> Path: + """Creating git repo, to make testing possible""" + repo_path = sphinx_base_dir + _ = subprocess.run(["git", "init"], cwd=repo_path, check=True) + _ = subprocess.run( + ["git", "config", "user.name", "Test User"], cwd=repo_path, check=True + ) + _ = subprocess.run( + ["git", "config", "user.email", "test@example.com"], cwd=repo_path, check=True + ) + _ = subprocess.run( + ["git", "remote", "add", "origin", "https://github.com/testorg/modulerepo.git"], + cwd=repo_path, + check=True, + ) + os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(repo_path) + return repo_path + + +@pytest.fixture(scope="session") +def create_demo_files(sphinx_base_dir: Path, git_repo_setup: Path): + repo_path = sphinx_base_dir + + # Create source files + source_dir = repo_path / "src" + source_dir.mkdir() + + _ = (source_dir / "module_a_impl.py").write_text(make_module_a_source()) + _ = (source_dir / "module_b_impl.py").write_text(make_module_b_source()) + + # Create docs directory + docs_dir = repo_path / "docs" + docs_dir.mkdir() + _ = (docs_dir / "index.rst").write_text(basic_needs()) + _ = (docs_dir / "conf.py").write_text(basic_conf()) + + # Create test.xml files + bazel_testdir = repo_path / "bazel-testlogs" / "src" + bazel_testdir.mkdir(parents=True) + _ = (bazel_testdir / "test.xml").write_text(make_test_xml()) + + curr_dir = Path(__file__).absolute().parent + _ = shutil.copyfile( + curr_dir / "expected_module_grouped.json", + repo_path / ".expected_module_grouped.json", + ) + + # Commit everything + _ = subprocess.run(["git", "add", "."], cwd=repo_path, check=True) + _ = subprocess.run( + ["git", "commit", "-m", "Initial commit with module test files"], + cwd=repo_path, + check=True, + ) + + +def make_module_a_source(): + return """ +# Module A implementation +# req-Id: MOD_REQ_1 +def module_a_function(): + pass + +# req-Id: MOD_REQ_2 +class ModuleAClass: + pass +""" + + +def make_module_b_source(): + return """ +# Module B implementation +# req-Id: MOD_REQ_1 +def module_b_function(): + pass + +# req-Id: MOD_REQ_3 +def another_module_b_function(): + pass +""" + + +def make_test_xml(): + return """ + + + + + + + + + + + + + + + + + + +""" + + +def basic_conf(): + return """ +extensions = [ + "sphinx_needs", + "score_source_code_linker", +] +needs_types = [ + dict( + directive="mod_req", + title="Module Requirement", + prefix="MOD_REQ_", + color="#BFD8D2", + style="node", + ), +] +needs_extra_options = ["source_code_link", "testlink"] +needs_extra_links = [{ + "option": "partially_verifies", + "incoming": "paritally_verified_by", + "outgoing": "paritally_verifies", + }, + { + "option": "fully_verifies", + "incoming": "fully_verified_by", + "outgoing": "fully_verifies", + }] +""" + + +def basic_needs(): + return """ +MODULE TESTING +============== + +.. mod_req:: Module Requirement 1 + :id: MOD_REQ_1 + :status: valid + +.. mod_req:: Module Requirement 2 + :id: MOD_REQ_2 + :status: open + +.. mod_req:: Module Requirement 3 + :id: MOD_REQ_3 + :status: open +""" + + +@pytest.fixture() +def sphinx_app_setup( + sphinx_base_dir: Path, create_demo_files: None, git_repo_setup: Path +) -> Callable[[], SphinxTestApp]: + def _create_app(): + base_dir = sphinx_base_dir + docs_dir = base_dir / "docs" + + original_cwd = None + with contextlib.suppress(FileNotFoundError): + original_cwd = os.getcwd() + + os.chdir(base_dir) + try: + return SphinxTestApp( + freshenv=True, + srcdir=docs_dir, + confdir=docs_dir, + outdir=sphinx_base_dir / "out", + buildername="html", + warningiserror=True, + ) + finally: + if original_cwd is not None: + with contextlib.suppress(FileNotFoundError, OSError): + os.chdir(original_cwd) + + return _create_app + + +def test_module_grouped_cache_generated( + sphinx_app_setup: Callable[[], SphinxTestApp], + sphinx_base_dir: Path, + git_repo_setup: Path, + create_demo_files: None, +): + """Happy path: Module grouped cache file is generated after Sphinx build""" + app = sphinx_app_setup() + try: + os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) + app.build() + + module_cache = app.outdir / "score_module_grouped_scl_cache.json" + assert module_cache.exists(), "Module grouped cache was not created" + + # Load and verify structure + loaded = load_module_source_links_json(module_cache) + assert isinstance(loaded, list) + assert len(loaded) > 0, "Module cache should contain at least one module" + + # Verify each item is a ModuleSourceLinks + for item in loaded: + assert hasattr(item, "module") + assert hasattr(item, "needs") + assert isinstance(item.module, ModuleInfo) + + finally: + app.cleanup() + + +def test_module_grouping_preserves_metadata( + sphinx_app_setup: Callable[[], SphinxTestApp], + sphinx_base_dir: Path, + git_repo_setup: Path, + create_demo_files: None, +): + """Happy path: Module metadata (name, hash, url) is preserved in cache""" + app = sphinx_app_setup() + try: + os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) + app.build() + + module_cache = app.outdir / "score_module_grouped_scl_cache.json" + loaded = load_module_source_links_json(module_cache) + + # Verify that each module has proper metadata + for module_links in loaded: + assert module_links.module.name is not None + assert isinstance(module_links.module.name, str) + # Hash and URL might be empty strings for local module + assert module_links.module.hash is not None + assert module_links.module.url is not None + + finally: + app.cleanup() + + +def test_module_grouping_multiple_needs_per_module( + sphinx_app_setup: Callable[[], SphinxTestApp], + sphinx_base_dir: Path, + git_repo_setup: Path, + create_demo_files: None, +): + """Happy path: Multiple needs from same module are grouped together""" + app = sphinx_app_setup() + try: + os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) + app.build() + + module_cache = app.outdir / "score_module_grouped_scl_cache.json" + loaded = load_module_source_links_json(module_cache) + + # Find the local_module (should have all 3 requirements) + local_module = None + for m in loaded: + if m.module.name == "local_module": + local_module = m + break + + assert local_module is not None, "local_module not found in grouped cache" + + # All 3 MOD_REQ should be in this module + need_ids = {need.need for need in local_module.needs} + assert ( + "MOD_REQ_1" in need_ids + or "MOD_REQ_2" in need_ids + or "MOD_REQ_3" in need_ids + ) + + finally: + app.cleanup() + + +def test_module_cache_json_format( + sphinx_app_setup: Callable[[], SphinxTestApp], + sphinx_base_dir: Path, + git_repo_setup: Path, + create_demo_files: None, +): + """Edge case: Module cache JSON has correct structure and excludes metadata from links""" + app = sphinx_app_setup() + try: + os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) + app.build() + + module_cache = app.outdir / "score_module_grouped_scl_cache.json" + + # Load as raw JSON to check structure + with open(module_cache) as f: + raw_json = json.load(f) + + assert isinstance(raw_json, list) + assert len(raw_json) > 0 + + # Check first module structure + first_module = raw_json[0] + assert "module" in first_module + assert "needs" in first_module + assert "name" in first_module["module"] + assert "hash" in first_module["module"] + assert "url" in first_module["module"] + + # Check that needlinks don't have metadata + if first_module["needs"]: + first_need = first_module["needs"][0] + if "links" in first_need: + if first_need["links"].get("CodeLinks"): + codelink = first_need["links"]["CodeLinks"][0] + assert "module_name" not in codelink, ( + "CodeLinks should not contain module_name metadata" + ) + assert "hash" not in codelink, ( + "CodeLinks should not contain hash metadata" + ) + assert "url" not in codelink, ( + "CodeLinks should not contain url metadata" + ) + + finally: + app.cleanup() + + +def test_module_cache_rebuilds_when_missing( + sphinx_app_setup: Callable[[], SphinxTestApp], + sphinx_base_dir: Path, + git_repo_setup: Path, + create_demo_files: None, +): + """Edge case: Module cache is regenerated if deleted""" + app = sphinx_app_setup() + try: + os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) + app.build() + + module_cache = app.outdir / "score_module_grouped_scl_cache.json" + assert module_cache.exists() + + # Delete the cache + module_cache.unlink() + assert not module_cache.exists() + + # Build again - should regenerate + app2 = sphinx_app_setup() + app2.build() + + assert module_cache.exists(), "Cache should be regenerated on rebuild" + + # Verify it's valid + loaded = load_module_source_links_json(module_cache) + assert len(loaded) > 0 + + app2.cleanup() + finally: + app.cleanup() + + +def test_module_grouping_with_golden_file( + sphinx_app_setup: Callable[[], SphinxTestApp], + sphinx_base_dir: Path, + git_repo_setup: Path, + create_demo_files: None, +): + """Happy path: Generated module cache matches expected golden file""" + app = sphinx_app_setup() + try: + os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) + app.build() + + module_cache = app.outdir / "score_module_grouped_scl_cache.json" + expected_file = sphinx_base_dir / ".expected_module_grouped.json" + + assert module_cache.exists() + assert expected_file.exists(), "Golden file not found" + + with open(module_cache) as f1: + actual = json.load(f1, object_hook=ModuleSourceLinks_JSON_Decoder) + with open(expected_file) as f2: + expected = json.load(f2, object_hook=ModuleSourceLinks_JSON_Decoder) + + assert len(actual) == len(expected), ( + f"Module count mismatch. Actual: {len(actual)}, Expected: {len(expected)}" + ) + + # Compare module by module + actual_by_name = {m.module.name: m for m in actual} + expected_by_name = {m.module.name: m for m in expected} + + assert set(actual_by_name.keys()) == set(expected_by_name.keys()), ( + f"Module names don't match. " + f"Actual: {set(actual_by_name.keys())}, " + f"Expected: {set(expected_by_name.keys())}" + ) + + for module_name in actual_by_name: + actual_module = actual_by_name[module_name] + expected_module = expected_by_name[module_name] + + assert actual_module.module.hash == expected_module.module.hash + assert actual_module.module.url == expected_module.module.url + assert len(actual_module.needs) == len(expected_module.needs) + + finally: + app.cleanup() diff --git a/src/extensions/score_source_code_linker/tests/test_testlink.py b/src/extensions/score_source_code_linker/tests/test_testlink.py index de6811183..c134c5ad6 100644 --- a/src/extensions/score_source_code_linker/tests/test_testlink.py +++ b/src/extensions/score_source_code_linker/tests/test_testlink.py @@ -10,8 +10,14 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* + +# ╓ ╖ +# ║ Some portions generated by Co-Pilot ║ +# ╙ ╜ + import json from pathlib import Path +import pytest # This depends on the `attribute_plugin` in our tooling repository from attribute_plugin import add_test_properties # type: ignore[import-untyped] @@ -23,6 +29,10 @@ DataOfTestCase, load_test_xml_parsed_json, store_test_xml_parsed_json, + store_data_of_test_case_json, + load_data_of_test_case_json, + DataOfTestCase_JSON_Encoder, + DataOfTestCase_JSON_Decoder, ) @@ -174,3 +184,221 @@ def test_store_and_load_testlinks_roundtrip(tmp_path: Path): assert reloaded == links for link in reloaded: assert isinstance(link, DataForTestLink) + + +def test_datafortestlink_to_dict_full(): + """Cover line 85: to_dict_full includes all fields""" + link = DataForTestLink( + name="test_full", + file=Path("src/test.py"), + line=10, + need="REQ_1", + verify_type="fully", + result="passed", + result_text="All good", + module_name="test_module", + hash="abc123", + url="https://github.com/test/repo", + ) + result = link.to_dict_full() + + assert result["module_name"] == "test_module" + assert result["hash"] == "abc123" + assert result["url"] == "https://github.com/test/repo" + assert result["name"] == "test_full" + + +def test_datafortestlink_equality_with_non_testlink(): + """Cover line 69: __eq__ with non-DataForTestLink returns NotImplemented""" + link = DataForTestLink( + name="test", + file=Path("src/test.py"), + line=10, + need="REQ_1", + verify_type="fully", + result="passed", + ) + + result = link.__eq__("not a testlink") + assert result is NotImplemented + + +# ──────────────────────[ DataOfTestCase ]────────────────────── + + +def test_dataoftestcase_check_verifies_fields_missing_both(): + """Cover lines 200-206: check_verifies_fields returns False when both are None""" + case = DataOfTestCase( + name="TC_Missing", + file="src/test.py", + line="10", + result="passed", + TestType="unit", + DerivationTechnique="manual", + result_text="", + PartiallyVerifies=None, + FullyVerifies=None, + module_name="mod", + hash="h", + url="u", + ) + + assert case.check_verifies_fields() is False + + +def test_dataoftestcase_is_valid_fails_on_none_field(): + """Cover lines 232-238: is_valid returns False when required field is None""" + case = DataOfTestCase( + name="TC_Invalid", + file=None, # Missing required field + line="10", + result="passed", + TestType="unit", + DerivationTechnique="manual", + result_text="", + PartiallyVerifies="REQ_1", + module_name="mod", + hash="h", + url="u", + ) + + assert case.is_valid() is False + + +def test_dataoftestcase_get_test_links_returns_empty_when_invalid(): + """Cover line 245: get_test_links returns [] when is_valid is False""" + case = DataOfTestCase( + name="TC_Invalid", + file=None, # Invalid + line="10", + result="passed", + TestType="unit", + DerivationTechnique="manual", + PartiallyVerifies="REQ_1", + ) + + links = case.get_test_links() + assert links == [] + + +def test_dataoftestcase_decoder_valid_dict(): + """Cover lines 314-324: Decoder reconstructs DataOfTestCase from valid dict""" + json_data = { + "name": "TC_01", + "file": "src/test.py", + "line": "10", + "result": "passed", + "TestType": "unit", + "DerivationTechnique": "manual", + "result_text": "Good", + "PartiallyVerifies": "REQ_1", + "FullyVerifies": "REQ_2", + "module_name": "mod", + "hash": "h", + "url": "u", + } + result = DataOfTestCase_JSON_Decoder(json_data) + + assert isinstance(result, DataOfTestCase) + assert result.name == "TC_01" + assert result.file == "src/test.py" + + +def test_dataoftestcase_decoder_non_testcase_dict(): + """Cover line 326: Decoder returns dict unchanged for non-TestCase data""" + json_data = {"some": "other", "data": "here"} + result = DataOfTestCase_JSON_Decoder(json_data) + assert result == json_data + + +def test_store_and_load_data_of_test_case_roundtrip(tmp_path: Path): + """Cover lines 376-386: store and load DataOfTestCase roundtrip""" + test_cases = [ + DataOfTestCase( + name="TC_A", + file="src/a.py", + line="5", + result="passed", + TestType="unit", + DerivationTechnique="manual", + result_text="OK", + PartiallyVerifies="REQ_A", + FullyVerifies=None, + module_name="mod_a", + hash="hash_a", + url="url_a", + ), + DataOfTestCase( + name="TC_B", + file="src/b.py", + line="15", + result="failed", + TestType="integration", + DerivationTechnique="auto", + result_text="Failed", + PartiallyVerifies=None, + FullyVerifies="REQ_B", + module_name="mod_b", + hash="hash_b", + url="url_b", + ), + ] + + test_file = tmp_path / "test_cases.json" + store_data_of_test_case_json(test_file, test_cases) + loaded = load_data_of_test_case_json(test_file) + + assert len(loaded) == 2 + assert loaded[0].name == "TC_A" + assert loaded[1].name == "TC_B" + + +def test_load_data_of_test_case_validates_list(tmp_path: Path): + """Cover line 380: Load validates result is a list""" + test_file = tmp_path / "not_list.json" + test_file.write_text('{"name": "TC_01"}') + + with pytest.raises(AssertionError, match="should be a list"): + _ = load_data_of_test_case_json(test_file) + + +def test_load_data_of_test_case_validates_items(tmp_path: Path): + """Cover line 383: Load validates all items are DataOfTestCase""" + test_file = tmp_path / "invalid_items.json" + test_file.write_text('[{"invalid": "structure"}]') + + with pytest.raises(AssertionError, match="should be TestCaseNeed objects"): + _ = load_data_of_test_case_json(test_file) + + +def test_dataoftestcase_encoder_fallback(): + """Cover line 299: Encoder falls back to parent for unknown types""" + encoder = DataOfTestCase_JSON_Encoder() + + # This should trigger the super().default() call + with pytest.raises(TypeError): + _ = encoder.default(object()) + + +def test_datafortestlink_encoder_fallback(): + """Cover line 103: DataForTestLink encoder falls back to parent for unknown types""" + from src.extensions.score_source_code_linker.testlink import ( + DataForTestLink_JSON_Encoder, + ) + + encoder = DataForTestLink_JSON_Encoder() + + # This should trigger the super().default() call + with pytest.raises(TypeError): + _ = encoder.default(object()) + + +def test_datafortestlink_decoder_non_testlink_dict(): + """Cover line 132: DataForTestLink decoder returns dict unchanged""" + from src.extensions.score_source_code_linker.testlink import ( + DataForTestLink_JSON_Decoder, + ) + + json_data = {"random": "data", "other": "stuff"} + result = DataForTestLink_JSON_Decoder(json_data) + assert result == json_data From 7cda52e70d4201881f9a1622d160b184d200155a Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Tue, 17 Mar 2026 10:23:53 +0100 Subject: [PATCH 17/34] Remove debug print statements --- .../generate_source_code_links_json.py | 5 ----- .../score_source_code_linker/module_source_links.py | 3 --- 2 files changed, 8 deletions(-) diff --git a/src/extensions/score_source_code_linker/generate_source_code_links_json.py b/src/extensions/score_source_code_linker/generate_source_code_links_json.py index 87e4aa554..dd0080450 100644 --- a/src/extensions/score_source_code_linker/generate_source_code_links_json.py +++ b/src/extensions/score_source_code_linker/generate_source_code_links_json.py @@ -58,11 +58,6 @@ def _extract_references_from_file( external/score_docs_as_code+/src/extensions/score_source_code_linker/testlink.py #FILE PATH NAME: testlink.py """ - print("ROOT: ", root) - print("FILE_PATH: ", file_path) - print("FILE_PATH_NAME: ", file_path_name) - output = subprocess.run(["ls", "-la", f"{root}"], capture_output=True) - print(output) assert root.is_absolute(), "Root path must be absolute" assert not file_path_name.is_absolute(), "File path must be relative to the root" # assert file_path.is_relative_to(root), ( diff --git a/src/extensions/score_source_code_linker/module_source_links.py b/src/extensions/score_source_code_linker/module_source_links.py index cc4e39165..103959bb6 100644 --- a/src/extensions/score_source_code_linker/module_source_links.py +++ b/src/extensions/score_source_code_linker/module_source_links.py @@ -113,9 +113,6 @@ def load_module_source_links_json(file: Path) -> list[ModuleSourceLinks]: assert isinstance(links, list), ( "The ModuleSourceLink json should be a list of ModuleSourceLink objects." ) - for link in links: - if not isinstance(link, ModuleSourceLinks): - print(f"Link not module_sourcelink: {link}") assert all(isinstance(link, ModuleSourceLinks) for link in links), ( "All items in module source link cache should be ModuleSourceLink objects." ) From 1d74ad06022eae2b1c94b9e819344e824d37d7f0 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Tue, 17 Mar 2026 10:58:47 +0100 Subject: [PATCH 18/34] Fixed linter warnings --- .../generate_source_code_links_json.py | 1 - .../module_source_links.py | 10 +++--- .../need_source_links.py | 2 +- .../score_source_code_linker/needlinks.py | 6 ++-- .../score_source_code_linker/testlink.py | 2 +- .../tests/test_codelink.py | 25 +++++++-------- .../tests/test_helpers.py | 12 +++---- .../tests/test_module_source_links.py | 8 +++-- .../test_module_source_links_integration.py | 32 ++++++++++--------- .../test_source_code_link_integration.py | 4 +-- .../tests/test_testlink.py | 9 +++--- 11 files changed, 55 insertions(+), 56 deletions(-) diff --git a/src/extensions/score_source_code_linker/generate_source_code_links_json.py b/src/extensions/score_source_code_linker/generate_source_code_links_json.py index dd0080450..cf9457c82 100644 --- a/src/extensions/score_source_code_linker/generate_source_code_links_json.py +++ b/src/extensions/score_source_code_linker/generate_source_code_links_json.py @@ -19,7 +19,6 @@ import os from pathlib import Path -import subprocess from sphinx_needs.logging import get_logger diff --git a/src/extensions/score_source_code_linker/module_source_links.py b/src/extensions/score_source_code_linker/module_source_links.py index 103959bb6..a24e279ac 100644 --- a/src/extensions/score_source_code_linker/module_source_links.py +++ b/src/extensions/score_source_code_linker/module_source_links.py @@ -40,7 +40,7 @@ class ModuleSourceLinks: class ModuleSourceLinks_JSON_Encoder(json.JSONEncoder): - def default(self, o: object): + def default(self, o: object) -> str | dict[str, Any]: if isinstance(o, Path): return str(o) # We do not want to save the metadata inside the codelink or testlink @@ -48,9 +48,9 @@ def default(self, o: object): # (hash, module_name, url) if isinstance(o, NeedLink | DataForTestLink): return o.to_dict_without_metadata() - # We need to split this up, otherwise the nested - # dictionaries won't get split up and we will not - # run into the 'to_dict_without_metadata' as + # We need to split this up, otherwise the nested + # dictionaries won't get split up and we will not + # run into the 'to_dict_without_metadata' as # everything will be converted to a normal dictionary if isinstance(o, ModuleSourceLinks): return { @@ -92,7 +92,7 @@ def ModuleSourceLinks_JSON_Decoder( def store_module_source_links_json( file: Path, source_code_links: list[ModuleSourceLinks] ): - # After `rm -rf _build` or on clean builds the directory does not exist, + # After `rm -rf _build` or on clean builds the directory does not exist, # so we need to create it. We create any folder that might be missing file.parent.mkdir(exist_ok=True, parents=True) with open(file, "w") as f: diff --git a/src/extensions/score_source_code_linker/need_source_links.py b/src/extensions/score_source_code_linker/need_source_links.py index a0b79d796..9a5d3c29e 100644 --- a/src/extensions/score_source_code_linker/need_source_links.py +++ b/src/extensions/score_source_code_linker/need_source_links.py @@ -82,7 +82,7 @@ def SourceCodeLinks_JSON_Decoder(d: dict[str, Any]) -> SourceCodeLinks | dict[st def store_source_code_links_combined_json( file: Path, source_code_links: list[SourceCodeLinks] ): - # After `rm -rf _build` or on clean builds the directory does not exist, + # After `rm -rf _build` or on clean builds the directory does not exist, # so we need to create it. We create any folder that might be missing file.parent.mkdir(exist_ok=True, parents=True) with open(file, "w") as f: diff --git a/src/extensions/score_source_code_linker/needlinks.py b/src/extensions/score_source_code_linker/needlinks.py index 588c95fef..52f424830 100644 --- a/src/extensions/score_source_code_linker/needlinks.py +++ b/src/extensions/score_source_code_linker/needlinks.py @@ -59,7 +59,7 @@ def __hash__(self): ) ) - def __eq__(self, other): + def __eq__(self, other: Any): if not isinstance(other, NeedLink): return NotImplemented return ( @@ -140,7 +140,7 @@ def store_source_code_links_with_metadata_json( """ payload: list[object] = [metadata, *needlist] - # After `rm -rf _build` or on clean builds the directory does not exist, + # After `rm -rf _build` or on clean builds the directory does not exist, # so we need to create it. We create any folder that might be missing file.parent.mkdir(exist_ok=True,parents=True) with open(file, "w", encoding="utf-8") as f: @@ -153,7 +153,7 @@ def store_source_code_links_json(file: Path, needlist: list[NeedLink]) -> None: [ needlink1, needlink2, ... ] """ - # After `rm -rf _build` or on clean builds the directory does not exist, + # After `rm -rf _build` or on clean builds the directory does not exist, # so we need to create it. We create any folder that might be missing file.parent.mkdir(exist_ok=True, parents=True) with open(file, "w", encoding="utf-8") as f: diff --git a/src/extensions/score_source_code_linker/testlink.py b/src/extensions/score_source_code_linker/testlink.py index 170bd0ad4..0dc1557f3 100644 --- a/src/extensions/score_source_code_linker/testlink.py +++ b/src/extensions/score_source_code_linker/testlink.py @@ -64,7 +64,7 @@ def __hash__(self): ) ) - def __eq__(self, other): + def __eq__(self, other: Any): if not isinstance(other, DataForTestLink): return NotImplemented return ( diff --git a/src/extensions/score_source_code_linker/tests/test_codelink.py b/src/extensions/score_source_code_linker/tests/test_codelink.py index e65adcb08..85277ef93 100644 --- a/src/extensions/score_source_code_linker/tests/test_codelink.py +++ b/src/extensions/score_source_code_linker/tests/test_codelink.py @@ -17,7 +17,6 @@ import json import os -from pkgutil import ModuleInfo import subprocess import tempfile from collections.abc import Generator @@ -29,8 +28,6 @@ # S-CORE plugin to allow for properties/attributes in xml # Enables Testlinking -from attribute_plugin import add_test_properties # type: ignore[import-untyped] - from sphinx_needs.data import NeedsInfoType, NeedsMutable from sphinx_needs.need_item import ( NeedItem, @@ -43,11 +40,11 @@ get_cache_filename, group_by_need, ) - +from src.extensions.score_source_code_linker.helpers import ( + get_github_link, +) from src.extensions.score_source_code_linker.module_source_links import ModuleInfo - from src.extensions.score_source_code_linker.needlinks import ( - DefaultNeedLink, MetaData, NeedLink, NeedLinkEncoder, @@ -62,11 +59,6 @@ get_current_git_hash, ) -from src.extensions.score_source_code_linker.helpers import ( - get_github_link_from_git, - get_github_link, -) - def test_need(**kwargs: Any) -> NeedItem: """Convenience function to create a NeedItem object with some defaults.""" @@ -626,6 +618,7 @@ def test_needlink_encoder_includes_metadata(): ) result = encoder.default(needlink) + assert isinstance(result, dict) assert result["module_name"] == "test_module" assert result["hash"] == "abc123" assert result["url"] == "https://github.com/test/repo" @@ -638,7 +631,7 @@ def test_needlink_encoder_includes_metadata(): def test_needlink_decoder_with_all_fields(): """Decoder reconstructs NeedLink with all fields""" - json_data = { + json_data: dict[str, Any] = { "file": "src/test.py", "line": 10, "tag": "# req-Id:", @@ -791,7 +784,9 @@ def test_load_with_metadata_invalid_items_after_metadata(tmp_path: Path): # ────────────────[ File Path Resolution Tests ]──────────────── -def test_load_resolves_relative_path_with_env_var(tmp_path: Path, monkeypatch): +def test_load_resolves_relative_path_with_env_var( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): """Edge case: Relative path is resolved using BUILD_WORKSPACE_DIRECTORY""" workspace = tmp_path / "workspace" workspace.mkdir() @@ -818,7 +813,9 @@ def test_load_resolves_relative_path_with_env_var(tmp_path: Path, monkeypatch): assert loaded[0].need == "REQ_1" -def test_load_with_metadata_resolves_relative_path(tmp_path: Path, monkeypatch): +def test_load_with_metadata_resolves_relative_path( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): """Edge case: load_with_metadata resolves relative paths using env var""" workspace = tmp_path / "workspace" workspace.mkdir() diff --git a/src/extensions/score_source_code_linker/tests/test_helpers.py b/src/extensions/score_source_code_linker/tests/test_helpers.py index 593253583..9ad02bc39 100644 --- a/src/extensions/score_source_code_linker/tests/test_helpers.py +++ b/src/extensions/score_source_code_linker/tests/test_helpers.py @@ -12,16 +12,12 @@ # ******************************************************************************* import json import os -from pathlib import Path +import subprocess import tempfile from collections.abc import Generator +from pathlib import Path import pytest -import subprocess - -from src.helper_lib import get_current_git_hash -from src.extensions.score_source_code_linker.module_source_links import ModuleInfo -from src.extensions.score_source_code_linker.needlinks import DefaultNeedLink from src.extensions.score_source_code_linker.helpers import ( get_github_link, @@ -29,7 +25,9 @@ parse_info_from_known_good, parse_module_name_from_path, ) - +from src.extensions.score_source_code_linker.module_source_links import ModuleInfo +from src.extensions.score_source_code_linker.needlinks import DefaultNeedLink +from src.helper_lib import get_current_git_hash # ╭──────────────────────────────────────────────────────────╮ # │ Tests for parse_module_name_from_path │ diff --git a/src/extensions/score_source_code_linker/tests/test_module_source_links.py b/src/extensions/score_source_code_linker/tests/test_module_source_links.py index 5119c3f0e..85acca4a6 100644 --- a/src/extensions/score_source_code_linker/tests/test_module_source_links.py +++ b/src/extensions/score_source_code_linker/tests/test_module_source_links.py @@ -10,8 +10,8 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -import json from pathlib import Path +from typing import Any import pytest @@ -31,7 +31,6 @@ from src.extensions.score_source_code_linker.needlinks import NeedLink from src.extensions.score_source_code_linker.testlink import DataForTestLink - """ ────────────────INFORMATION─────────────── @@ -64,6 +63,7 @@ def test_json_encoder_removes_metadata_from_needlink(): ) result = encoder.default(needlink) + assert isinstance(result, dict) assert "module_name" not in result assert "url" not in result assert "hash" not in result @@ -88,6 +88,8 @@ def test_json_encoder_removes_metadata_from_testlink(): ) result = encoder.default(testlink) + + assert isinstance(result, dict) assert "module_name" not in result assert "url" not in result assert "hash" not in result @@ -110,7 +112,7 @@ def test_json_encoder_converts_path_to_string(): def test_json_decoder_reconstructs_module_source_links(): """Happy path: Valid JSON dict is decoded into ModuleSourceLinks""" - json_data = { + json_data: dict[str,Any] = { "module": {"name": "test_module", "hash": "hash1", "url": "url1"}, "needs": [ { diff --git a/src/extensions/score_source_code_linker/tests/test_module_source_links_integration.py b/src/extensions/score_source_code_linker/tests/test_module_source_links_integration.py index 60c217c25..d209a7d5c 100644 --- a/src/extensions/score_source_code_linker/tests/test_module_source_links_integration.py +++ b/src/extensions/score_source_code_linker/tests/test_module_source_links_integration.py @@ -27,8 +27,6 @@ ModuleSourceLinks_JSON_Decoder, load_module_source_links_json, ) -from src.helper_lib import find_ws_root - """ ────────────────INFORMATION─────────────── @@ -131,6 +129,7 @@ def another_module_b_function(): def make_test_xml(): + # ruff: noqa: E501 (start) return """ @@ -151,6 +150,7 @@ def make_test_xml(): """ + # ruff: noqa: E501 (finish) def basic_conf(): @@ -330,7 +330,10 @@ def test_module_cache_json_format( git_repo_setup: Path, create_demo_files: None, ): - """Edge case: Module cache JSON has correct structure and excludes metadata from links""" + """ + Module cache JSON has correct + structure and excludes metadata from links + """ app = sphinx_app_setup() try: os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) @@ -356,18 +359,17 @@ def test_module_cache_json_format( # Check that needlinks don't have metadata if first_module["needs"]: first_need = first_module["needs"][0] - if "links" in first_need: - if first_need["links"].get("CodeLinks"): - codelink = first_need["links"]["CodeLinks"][0] - assert "module_name" not in codelink, ( - "CodeLinks should not contain module_name metadata" - ) - assert "hash" not in codelink, ( - "CodeLinks should not contain hash metadata" - ) - assert "url" not in codelink, ( - "CodeLinks should not contain url metadata" - ) + if "links" in first_need and first_need["links"].get("CodeLinks"): + codelink = first_need["links"]["CodeLinks"][0] + assert "module_name" not in codelink, ( + "CodeLinks should not contain module_name metadata" + ) + assert "hash" not in codelink, ( + "CodeLinks should not contain hash metadata" + ) + assert "url" not in codelink, ( + "CodeLinks should not contain url metadata" + ) finally: app.cleanup() diff --git a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py index e49d856da..b6e1ded0b 100644 --- a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py +++ b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py @@ -25,8 +25,9 @@ from sphinx.testing.util import SphinxTestApp from sphinx_needs.data import SphinxNeedsData -from src.extensions.score_source_code_linker.needlinks import NeedLink +from src.extensions.score_source_code_linker.helpers import get_github_link from src.extensions.score_source_code_linker.module_source_links import ModuleInfo +from src.extensions.score_source_code_linker.needlinks import NeedLink from src.extensions.score_source_code_linker.testlink import ( DataForTestLink, DataForTestLink_JSON_Decoder, @@ -38,7 +39,6 @@ SourceCodeLinks_TEST_JSON_Decoder, ) from src.helper_lib import find_ws_root, get_github_base_url -from src.extensions.score_source_code_linker.helpers import get_github_link @pytest.fixture(scope="session") diff --git a/src/extensions/score_source_code_linker/tests/test_testlink.py b/src/extensions/score_source_code_linker/tests/test_testlink.py index c134c5ad6..1916f5c0d 100644 --- a/src/extensions/score_source_code_linker/tests/test_testlink.py +++ b/src/extensions/score_source_code_linker/tests/test_testlink.py @@ -17,6 +17,7 @@ import json from pathlib import Path + import pytest # This depends on the `attribute_plugin` in our tooling repository @@ -27,12 +28,12 @@ DataForTestLink_JSON_Decoder, DataForTestLink_JSON_Encoder, DataOfTestCase, + DataOfTestCase_JSON_Decoder, + DataOfTestCase_JSON_Encoder, + load_data_of_test_case_json, load_test_xml_parsed_json, - store_test_xml_parsed_json, store_data_of_test_case_json, - load_data_of_test_case_json, - DataOfTestCase_JSON_Encoder, - DataOfTestCase_JSON_Decoder, + store_test_xml_parsed_json, ) From ddfa34c6b6f5b31d6a602bec8f2752779183dc81 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Tue, 17 Mar 2026 11:02:46 +0100 Subject: [PATCH 19/34] Fix copyright year --- src/extensions/score_source_code_linker/tests/test_codelink.py | 2 +- src/extensions/score_source_code_linker/tests/test_testlink.py | 2 +- .../score_source_code_linker/tests/test_xml_parser.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/extensions/score_source_code_linker/tests/test_codelink.py b/src/extensions/score_source_code_linker/tests/test_codelink.py index 85277ef93..26a9b9058 100644 --- a/src/extensions/score_source_code_linker/tests/test_codelink.py +++ b/src/extensions/score_source_code_linker/tests/test_codelink.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2025-2026 Contributors to the Eclipse Foundation +# Copyright (c) 2026 Contributors to the Eclipse Foundation # # See the NOTICE file(s) distributed with this work for additional # information regarding copyright ownership. diff --git a/src/extensions/score_source_code_linker/tests/test_testlink.py b/src/extensions/score_source_code_linker/tests/test_testlink.py index 1916f5c0d..7e49b8983 100644 --- a/src/extensions/score_source_code_linker/tests/test_testlink.py +++ b/src/extensions/score_source_code_linker/tests/test_testlink.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2025-2026 Contributors to the Eclipse Foundation +# Copyright (c) 2026 Contributors to the Eclipse Foundation # # See the NOTICE file(s) distributed with this work for additional # information regarding copyright ownership. diff --git a/src/extensions/score_source_code_linker/tests/test_xml_parser.py b/src/extensions/score_source_code_linker/tests/test_xml_parser.py index c283aada5..76958122b 100644 --- a/src/extensions/score_source_code_linker/tests/test_xml_parser.py +++ b/src/extensions/score_source_code_linker/tests/test_xml_parser.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2025-2026 Contributors to the Eclipse Foundation +# Copyright (c) 2026 Contributors to the Eclipse Foundation # # See the NOTICE file(s) distributed with this work for additional # information regarding copyright ownership. From 6230c47a4704153e25742219e991a3d4420195a2 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Tue, 17 Mar 2026 11:49:45 +0100 Subject: [PATCH 20/34] Fix scl finding test needs --- .../module_source_links.py | 4 +- .../tests/expected_module_grouped.json | 16 +- .../tests/test_codelink.py | 76 +++++----- .../tests/test_module_source_links.py | 138 +++++++++++++++--- .../test_module_source_links_integration.py | 8 +- 5 files changed, 167 insertions(+), 75 deletions(-) diff --git a/src/extensions/score_source_code_linker/module_source_links.py b/src/extensions/score_source_code_linker/module_source_links.py index a24e279ac..1521f6868 100644 --- a/src/extensions/score_source_code_linker/module_source_links.py +++ b/src/extensions/score_source_code_linker/module_source_links.py @@ -39,7 +39,7 @@ class ModuleSourceLinks: needs: list[SourceCodeLinks] = field(default_factory=list) -class ModuleSourceLinks_JSON_Encoder(json.JSONEncoder): +class ModuleSourceLinks_TEST_JSON_Encoder(json.JSONEncoder): def default(self, o: object) -> str | dict[str, Any]: if isinstance(o, Path): return str(o) @@ -99,7 +99,7 @@ def store_module_source_links_json( json.dump( source_code_links, f, - cls=ModuleSourceLinks_JSON_Encoder, + cls=ModuleSourceLinks_TEST_JSON_Encoder, indent=2, ensure_ascii=False, ) diff --git a/src/extensions/score_source_code_linker/tests/expected_module_grouped.json b/src/extensions/score_source_code_linker/tests/expected_module_grouped.json index fd1ebfd79..ef91784bf 100644 --- a/src/extensions/score_source_code_linker/tests/expected_module_grouped.json +++ b/src/extensions/score_source_code_linker/tests/expected_module_grouped.json @@ -13,16 +13,16 @@ { "file": "src/module_a_impl.py", "line": 3, - "tag": "# req-Id:", + "tag": "#-----req-Id:", "need": "MOD_REQ_1", - "full_line": "# req-Id: MOD_REQ_1" + "full_line": "#-----req-Id: MOD_REQ_1" }, { "file": "src/module_b_impl.py", "line": 3, - "tag": "# req-Id:", + "tag": "#-----req-Id:", "need": "MOD_REQ_1", - "full_line": "# req-Id: MOD_REQ_1" + "full_line": "#-----req-Id: MOD_REQ_1" } ], "TestLinks": [ @@ -45,9 +45,9 @@ { "file": "src/module_a_impl.py", "line": 7, - "tag": "# req-Id:", + "tag": "#-----req-Id:", "need": "MOD_REQ_2", - "full_line": "# req-Id: MOD_REQ_2" + "full_line": "#-----req-Id: MOD_REQ_2" } ], "TestLinks": [] @@ -60,9 +60,9 @@ { "file": "src/module_b_impl.py", "line": 7, - "tag": "# req-Id:", + "tag": "#-----req-Id:", "need": "MOD_REQ_3", - "full_line": "# req-Id: MOD_REQ_3" + "full_line": "#-----req-Id: MOD_REQ_3" } ], "TestLinks": [ diff --git a/src/extensions/score_source_code_linker/tests/test_codelink.py b/src/extensions/score_source_code_linker/tests/test_codelink.py index 26a9b9058..1328e1fa2 100644 --- a/src/extensions/score_source_code_linker/tests/test_codelink.py +++ b/src/extensions/score_source_code_linker/tests/test_codelink.py @@ -564,9 +564,9 @@ def test_needlink_to_dict_without_metadata(): needlink = NeedLink( file=Path("src/test.py"), line=10, - tag="# req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#"+" req-Id: REQ_1", module_name="test_module", url="https://github.com/test/repo", hash="abc123", @@ -585,9 +585,9 @@ def test_needlink_to_dict_full(): needlink = NeedLink( file=Path("src/test.py"), line=10, - tag="# req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#" + " req-Id: REQ_1", module_name="test_module", url="https://github.com/test/repo", hash="abc123", @@ -609,9 +609,9 @@ def test_needlink_encoder_includes_metadata(): needlink = NeedLink( file=Path("src/test.py"), line=10, - tag="# req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#" + " req-Id: REQ_1", module_name="test_module", url="https://github.com/test/repo", hash="abc123", @@ -634,9 +634,9 @@ def test_needlink_decoder_with_all_fields(): json_data: dict[str, Any] = { "file": "src/test.py", "line": 10, - "tag": "# req-Id:", + "tag":"#" + " req-Id:", "need": "REQ_1", - "full_line": "# req-Id: REQ_1", + "full_line": "#" +" req-Id: REQ_1", "module_name": "test_module", "hash": "abc123", "url": "https://github.com/test/repo", @@ -665,9 +665,9 @@ def test_store_and_load_source_code_links(tmp_path: Path): NeedLink( file=Path("src/impl.py"), line=42, - tag="# req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#" + " req-Id: REQ_1", module_name="mod", url="url", hash="hash", @@ -715,9 +715,9 @@ def test_store_and_load_with_metadata(tmp_path: Path): NeedLink( file=Path("src/impl.py"), line=10, - tag="# req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#"+" req-Id: REQ_1", module_name="", # Will be filled from metadata url="", hash="", @@ -725,9 +725,9 @@ def test_store_and_load_with_metadata(tmp_path: Path): NeedLink( file=Path("src/impl2.py"), line=20, - tag="# req-Id:", + tag="#" + " req-Id:", need="REQ_2", - full_line="# req-Id: REQ_2", + full_line="#"+" req-Id: REQ_2", module_name="", url="", hash="", @@ -753,9 +753,9 @@ def test_load_with_metadata_missing_metadata_dict(tmp_path: Path): NeedLink( file=Path("src/test.py"), line=1, - tag="# req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#" +" req-Id: REQ_1", ) ] @@ -795,9 +795,9 @@ def test_load_resolves_relative_path_with_env_var( NeedLink( file=Path("src/test.py"), line=1, - tag="# req-Id:", + tag="#"+" req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#"+" req-Id: REQ_1", ) ] @@ -829,9 +829,9 @@ def test_load_with_metadata_resolves_relative_path( NeedLink( file=Path("src/test.py"), line=1, - tag="# req-Id:", + tag="#"+" req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#"+" req-Id: REQ_1", ) ] @@ -854,9 +854,9 @@ def test_roundtrip_standard_format(tmp_path: Path): NeedLink( file=Path("src/file1.py"), line=10, - tag="# req-Id:", + tag="#"+" req-Id:", need="REQ_A", - full_line="# req-Id: REQ_A", + full_line="#"+" req-Id: REQ_A", module_name="mod_a", url="url_a", hash="hash_a", @@ -864,9 +864,9 @@ def test_roundtrip_standard_format(tmp_path: Path): NeedLink( file=Path("src/file2.py"), line=20, - tag="# req-Id:", + tag="#"+" req-Id:", need="REQ_B", - full_line="# req-Id: REQ_B", + full_line="#"+" req-Id: REQ_B", module_name="mod_b", url="url_b", hash="hash_b", @@ -893,16 +893,16 @@ def test_roundtrip_metadata_format_applies_metadata(tmp_path: Path): NeedLink( file=Path("src/f1.py"), line=5, - tag="# req-Id:", + tag="#"+" req-Id:", need="R1", - full_line="# req-Id: R1", + full_line="#"+" req-Id: R1", ), NeedLink( file=Path("src/f2.py"), line=15, - tag="# req-Id:", + tag="#"+" req-Id:", need="R2", - full_line="# req-Id: R2", + full_line="#"+" req-Id: R2", ), ] @@ -940,9 +940,9 @@ def test_json_format_with_metadata_has_separate_dict(tmp_path: Path): needlink = NeedLink( file=Path("src/test.py"), line=1, - tag="# req-Id:", + tag="#"+" req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#"+" req-Id: REQ_1", ) test_file = tmp_path / "metadata_format.json" @@ -967,9 +967,9 @@ def test_needlink_equality_same_values(): link1 = NeedLink( file=Path("src/test.py"), line=10, - tag="# req-Id:", + tag="#"+" req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#"+" req-Id: REQ_1", module_name="mod", url="url", hash="hash", @@ -977,9 +977,9 @@ def test_needlink_equality_same_values(): link2 = NeedLink( file=Path("src/test.py"), line=10, - tag="# req-Id:", + tag="#"+" req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#"+" req-Id: REQ_1", module_name="mod", url="url", hash="hash", @@ -994,16 +994,16 @@ def test_needlink_inequality_different_values(): link1 = NeedLink( file=Path("src/test.py"), line=10, - tag="# req-Id:", + tag="#"+" req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#"+" req-Id: REQ_1", ) link2 = NeedLink( file=Path("src/test.py"), line=20, # Different line - tag="# req-Id:", + tag="#"+" req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#"+" req-Id: REQ_1", ) assert link1 != link2 diff --git a/src/extensions/score_source_code_linker/tests/test_module_source_links.py b/src/extensions/score_source_code_linker/tests/test_module_source_links.py index 85acca4a6..0da7594b8 100644 --- a/src/extensions/score_source_code_linker/tests/test_module_source_links.py +++ b/src/extensions/score_source_code_linker/tests/test_module_source_links.py @@ -12,6 +12,8 @@ # ******************************************************************************* from pathlib import Path from typing import Any +import json +from dataclasses import asdict import pytest @@ -19,7 +21,6 @@ ModuleInfo, ModuleSourceLinks, ModuleSourceLinks_JSON_Decoder, - ModuleSourceLinks_JSON_Encoder, group_needs_by_module, load_module_source_links_json, store_module_source_links_json, @@ -48,15 +49,107 @@ # ╰──────────────────────────────────────────────────────────╯ +def encode_comment(s: str) -> str: + return s.replace(" ", "-----", 1) + + +def decode_comment(s: str) -> str: + return s.replace("-----", " ", 1) + + +def SourceCodeLinks_TEST_JSON_Decoder( + d: dict[str, Any], +) -> SourceCodeLinks | dict[str, Any]: + if "need" in d and "links" in d: + links = d["links"] + + # Decode CodeLinks + code_links = [] + for cl in links.get("CodeLinks", []): + # Decode the tag and full_line fields + if "tag" in cl: + cl["tag"] = decode_comment(cl["tag"]) + if "full_line" in cl: + cl["full_line"] = decode_comment(cl["full_line"]) + code_links.append(NeedLink(**cl)) + + # Decode TestLinks + return SourceCodeLinks( + need=d["need"], + links=NeedSourceLinks( + CodeLinks=code_links, + TestLinks=[DataForTestLink(**tl) for tl in links.get("TestLinks", [])], + ), + ) + return d + + +class ModuleSourceLinks_TEST_JSON_Encoder(json.JSONEncoder): + def default(self, o: object) -> str | dict[str, Any]: + if isinstance(o, Path): + return str(o) + # We do not want to save the metadata inside the codelink or testlink + # As we save this already in a structure above it + # (hash, module_name, url) + if isinstance(o, NeedLink | DataForTestLink): + d = o.to_dict_without_metadata() + tag = d.get("tag", "") + full_line = d.get("full_line", "") + assert isinstance(tag, str) + assert isinstance(full_line, str) + d["tag"] = encode_comment(tag) + d["full_line"] = encode_comment(full_line) + return d + # We need to split this up, otherwise the nested + # dictionaries won't get split up and we will not + # run into the 'to_dict_without_metadata' as + # everything will be converted to a normal dictionary + if isinstance(o, ModuleSourceLinks): + return { + "module": asdict(o.module), + "needs": o.needs, # Let the encoder handle the list + } + if isinstance(o, SourceCodeLinks): + return { + "need": o.need, + "links": o.links, + } + if isinstance(o, NeedSourceLinks): + return { + "CodeLinks": o.CodeLinks, + "TestLinks": o.TestLinks, + } + return super().default(o) + + +def ModuleSourceLinks_TEST_JSON_Decoder( + d: dict[str, Any], +) -> ModuleSourceLinks | dict[str, Any]: + if "module" in d and "needs" in d: + module = d["module"] + needs = d["needs"] + return ModuleSourceLinks( + module=ModuleInfo( + name=module.get("name"), + hash=module.get("hash"), + url=module.get("url"), + ), + # We know this can only be list[SourceCodeLinks] and nothing else + # Therefore => we ignore the type error here + needs=[SourceCodeLinks_TEST_JSON_Decoder(need) for need in needs], # type: ignore + ) + return d + + def test_json_encoder_removes_metadata_from_needlink(): """Happy path: NeedLink metadata fields are excluded from JSON output""" - encoder = ModuleSourceLinks_JSON_Encoder() + encoder = ModuleSourceLinks_TEST_JSON_Encoder() needlink = NeedLink( file=Path("src/test.py"), line=10, - tag="# req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#" + " req-Id: REQ_1", module_name="test_module", url="https://github.com/test/repo", hash="abc123", @@ -73,7 +166,7 @@ def test_json_encoder_removes_metadata_from_needlink(): def test_json_encoder_removes_metadata_from_testlink(): """Happy path: DataForTestLink metadata fields are excluded from JSON output""" - encoder = ModuleSourceLinks_JSON_Encoder() + encoder = ModuleSourceLinks_TEST_JSON_Encoder() testlink = DataForTestLink( name="test_something", file=Path("src/test_file.py"), @@ -88,7 +181,6 @@ def test_json_encoder_removes_metadata_from_testlink(): ) result = encoder.default(testlink) - assert isinstance(result, dict) assert "module_name" not in result assert "url" not in result @@ -99,7 +191,7 @@ def test_json_encoder_removes_metadata_from_testlink(): def test_json_encoder_converts_path_to_string(): """Happy path: Path objects are converted to strings""" - encoder = ModuleSourceLinks_JSON_Encoder() + encoder = ModuleSourceLinks_TEST_JSON_Encoder() result = encoder.default(Path("/test/path/file.py")) assert result == "/test/path/file.py" assert isinstance(result, str) @@ -112,7 +204,7 @@ def test_json_encoder_converts_path_to_string(): def test_json_decoder_reconstructs_module_source_links(): """Happy path: Valid JSON dict is decoded into ModuleSourceLinks""" - json_data: dict[str,Any] = { + json_data: dict[str, Any] = { "module": {"name": "test_module", "hash": "hash1", "url": "url1"}, "needs": [ { @@ -148,9 +240,9 @@ def test_store_and_load_roundtrip(tmp_path: Path): needlink = NeedLink( file=Path("src/test.py"), line=10, - tag="# req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#" + " req-Id: REQ_1", module_name="test_module", url="url1", hash="hash1", @@ -219,9 +311,9 @@ def test_group_needs_single_module_with_codelinks(): needlink1 = NeedLink( file=Path("src/file1.py"), line=10, - tag="# req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#" + " req-Id: REQ_1", module_name="shared_module", url="https://github.com/test/repo", hash="hash1", @@ -229,9 +321,9 @@ def test_group_needs_single_module_with_codelinks(): needlink2 = NeedLink( file=Path("src/file2.py"), line=20, - tag="# req-Id:", + tag="#" + " req-Id:", need="REQ_2", - full_line="# req-Id: REQ_2", + full_line="#" + " req-Id: REQ_2", module_name="shared_module", url="https://github.com/test/repo", hash="hash1", @@ -259,9 +351,9 @@ def test_group_needs_multiple_modules(): needlink_a = NeedLink( file=Path("src/a.py"), line=10, - tag="# req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#" + " req-Id: REQ_1", module_name="module_a", url="https://github.com/a/repo", hash="hash_a", @@ -269,9 +361,9 @@ def test_group_needs_multiple_modules(): needlink_b = NeedLink( file=Path("src/b.py"), line=20, - tag="# req-Id:", + tag="#" + " req-Id:", need="REQ_2", - full_line="# req-Id: REQ_2", + full_line="#" + " req-Id: REQ_2", module_name="module_b", url="https://github.com/b/repo", hash="hash_b", @@ -409,9 +501,9 @@ def test_group_needs_skips_needs_without_links(): NeedLink( file=Path("src/test.py"), line=10, - tag="# req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#" + " req-Id: REQ_1", module_name="module_a", url="url1", hash="hash1", @@ -429,7 +521,7 @@ def test_group_needs_skips_needs_without_links(): assert len(result) == 1 assert result[0].needs[0].need == "REQ_1" - assert result[0].needs[0].links.CodeLinks[0].full_line == "# req-Id: REQ_1" + assert result[0].needs[0].links.CodeLinks[0].full_line == "#" + " req-Id: REQ_1" def test_group_needs_mixed_codelinks_and_testlinks(): @@ -437,9 +529,9 @@ def test_group_needs_mixed_codelinks_and_testlinks(): needlink = NeedLink( file=Path("src/impl.py"), line=5, - tag="# req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="# req-Id: REQ_1", + full_line="#" + " req-Id: REQ_1", module_name="module_a", url="https://github.com/test/repo", hash="hash1", diff --git a/src/extensions/score_source_code_linker/tests/test_module_source_links_integration.py b/src/extensions/score_source_code_linker/tests/test_module_source_links_integration.py index d209a7d5c..895a8eb56 100644 --- a/src/extensions/score_source_code_linker/tests/test_module_source_links_integration.py +++ b/src/extensions/score_source_code_linker/tests/test_module_source_links_integration.py @@ -105,11 +105,11 @@ def create_demo_files(sphinx_base_dir: Path, git_repo_setup: Path): def make_module_a_source(): return """ # Module A implementation -# req-Id: MOD_REQ_1 +# """+"""req-Id: MOD_REQ_1 def module_a_function(): pass -# req-Id: MOD_REQ_2 +# """+"""req-Id: MOD_REQ_2 class ModuleAClass: pass """ @@ -118,11 +118,11 @@ class ModuleAClass: def make_module_b_source(): return """ # Module B implementation -# req-Id: MOD_REQ_1 +# """+"""req-Id: MOD_REQ_1 def module_b_function(): pass -# req-Id: MOD_REQ_3 +# """+"""req-Id: MOD_REQ_3 def another_module_b_function(): pass """ From a8ce766ad5ac6fa027041a7a5d14363d7d8e09c7 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Tue, 17 Mar 2026 11:50:28 +0100 Subject: [PATCH 21/34] Formatting --- .../tests/test_module_source_links.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/extensions/score_source_code_linker/tests/test_module_source_links.py b/src/extensions/score_source_code_linker/tests/test_module_source_links.py index 0da7594b8..293cdc76e 100644 --- a/src/extensions/score_source_code_linker/tests/test_module_source_links.py +++ b/src/extensions/score_source_code_linker/tests/test_module_source_links.py @@ -10,10 +10,10 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -from pathlib import Path -from typing import Any import json from dataclasses import asdict +from pathlib import Path +from typing import Any import pytest From 740b019460819e30a8ed2250954a04b2a011a826 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Tue, 17 Mar 2026 15:00:57 +0100 Subject: [PATCH 22/34] Fix: Testlinks didn't clean filepath properly --- .../tests/test_xml_parser.py | 52 +++++++++++++++++++ .../score_source_code_linker/xml_parser.py | 42 ++++++++++++++- 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/src/extensions/score_source_code_linker/tests/test_xml_parser.py b/src/extensions/score_source_code_linker/tests/test_xml_parser.py index 76958122b..b33000121 100644 --- a/src/extensions/score_source_code_linker/tests/test_xml_parser.py +++ b/src/extensions/score_source_code_linker/tests/test_xml_parser.py @@ -10,6 +10,7 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* +# Some portions generated by CoPilot """ Tests for the xml_parser.py file. Keep in mind that this is with the 'assertions' inside xml_parser disabled so far. @@ -326,3 +327,54 @@ def test_short_hash_consistency_and_format(): assert h1 == h2 assert h1.isalpha() assert len(h1) == 5 + + +# ─────────────[ Boilerplate generated by CoPilot ]───────────── + + +def test_clean_test_file_name_local_path(): + raw_path = Path( + "/home/root/folder/docs-as-code/bazel-testlogs/src/extensions/score_any_folder/score_any_folder_tests/test.xml" + ) + result = xml_parser.clean_test_file_name(raw_path) + assert result == Path( + "src/extensions/score_any_folder/score_any_folder_tests/test.xml" + ) + + +def test_clean_test_file_name_combo_path(): + raw_path = Path( + "/root/bazel-testlogs/external/score_docs_as_code+/src/extensions/score_any_folder/score_any_folder_tests/test.xml" + ) + result = xml_parser.clean_test_file_name(raw_path) + assert result == Path( + "external/score_docs_as_code+/src/extensions/score_any_folder/score_any_folder_tests/test.xml" + ) + + +def test_clean_test_file_name_tests_report_path(): + raw_path = Path("/some/path/tests-report/test-report/unit/test_module.xml") + result = xml_parser.clean_test_file_name(raw_path) + assert result == Path("unit/test_module.xml") + + +def test_clean_test_file_name_nested_bazel_testlogs(): + raw_path = Path("/deeply/nested/bazel-testlogs/bazel-testlogs/final/path.xml") + result = xml_parser.clean_test_file_name(raw_path) + assert result == Path("final/path.xml") + + +def test_clean_test_file_name_invalid_path_raises_error(): + raw_path = Path("/invalid/path/without/markers/test.xml") + with pytest.raises( + ValueError, match="Filepath does not have 'bazel-testlogs' nor 'test-report'" + ): + xml_parser.clean_test_file_name(raw_path) + + +def test_clean_test_file_name_empty_path_raises_error(): + raw_path = Path("") + with pytest.raises( + ValueError, match="Filepath does not have 'bazel-testlogs' nor 'test-report'" + ): + xml_parser.clean_test_file_name(raw_path) diff --git a/src/extensions/score_source_code_linker/xml_parser.py b/src/extensions/score_source_code_linker/xml_parser.py index 7347429ce..af80c76be 100644 --- a/src/extensions/score_source_code_linker/xml_parser.py +++ b/src/extensions/score_source_code_linker/xml_parser.py @@ -53,7 +53,31 @@ logger.setLevel("DEBUG") -def get_metadata_from_test_path(filepath: Path) -> MetaData: +def clean_test_file_name(raw_filepath: Path) -> Path: + """ + incoming path: + `local` + /docs-as-code/bazel-testlogs/src/extensions/score_any_folder/score_any_folder_tests/test.xml + `combo` + /bazel-testlogs/external/score_docs_as_code+/src/extensions/score_any_folder/score_any_folder_tests/test.xml + + outgoing path: + `local` + src/extensions/score_any_folder/score_any_folder_tests/test.xml + `combo` + external/score_docs_as_code+/src/extensions/score_any_folder/score_any_folder_tests/test.xml + """ + if "bazel-testlogs" in str(raw_filepath): + return Path(str(raw_filepath).split("bazel-testlogs/")[-1]) + elif "tests-report" in str(raw_filepath): + return Path(str(raw_filepath).split("test-report/")[-1]) + else: + raise ValueError( + f"Filepath does not have 'bazel-testlogs' nor 'test-report'. Filepath: {raw_filepath}" + ) + + +def get_metadata_from_test_path(raw_filepath: Path) -> MetaData: """ Will parse out the metadata from the testpath. If test is local then the metadata will be: @@ -70,9 +94,23 @@ def get_metadata_from_test_path(filepath: Path) -> MetaData: "hash": "c1207676afe6cafd25c35d420e73279a799515d8", "url": "https://github.com/eclipse-score/docs-as-code" + The file name passed into here is: + + For combo builds: something like: + /bazel-testlogs/external/score_docs_as_code+/ + src/extensions/score_any_folder/score_any_folder_tests/test.xml + + For local builds: + /bazel-testlogs/src/extensions/score_any_folder/score_any_folder_tests/test.xml + + Therefore will we 'clean' it before passing it to the parse_module func. + Removing everything up to and including 'bazel-testlogs' or 'tests-report' """ + #print("THIs IS FILEPATH IN GET MD FROm TestPATH: ", raw_filepath) known_good_json = os.environ.get("KNOWN_GOOD_JSON") - module_name = parse_module_name_from_path(filepath) + clean_filepath = clean_test_file_name(raw_filepath) + #print(f"This is the cleaned filepath: {clean_filepath}") + module_name = parse_module_name_from_path(clean_filepath) md: MetaData = { "module_name": module_name, "hash": "", From 691528337508f45e6eda731696235b8a9c05e8f9 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Tue, 17 Mar 2026 15:06:35 +0100 Subject: [PATCH 23/34] Linting errors --- .../score_source_code_linker/xml_parser.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/extensions/score_source_code_linker/xml_parser.py b/src/extensions/score_source_code_linker/xml_parser.py index af80c76be..d44f0854c 100644 --- a/src/extensions/score_source_code_linker/xml_parser.py +++ b/src/extensions/score_source_code_linker/xml_parser.py @@ -69,12 +69,12 @@ def clean_test_file_name(raw_filepath: Path) -> Path: """ if "bazel-testlogs" in str(raw_filepath): return Path(str(raw_filepath).split("bazel-testlogs/")[-1]) - elif "tests-report" in str(raw_filepath): + if "tests-report" in str(raw_filepath): return Path(str(raw_filepath).split("test-report/")[-1]) - else: - raise ValueError( - f"Filepath does not have 'bazel-testlogs' nor 'test-report'. Filepath: {raw_filepath}" - ) + raise ValueError( + "Filepath does not have 'bazel-testlogs' nor " + f"'test-report'. Filepath: {raw_filepath}" + ) def get_metadata_from_test_path(raw_filepath: Path) -> MetaData: @@ -101,15 +101,15 @@ def get_metadata_from_test_path(raw_filepath: Path) -> MetaData: src/extensions/score_any_folder/score_any_folder_tests/test.xml For local builds: - /bazel-testlogs/src/extensions/score_any_folder/score_any_folder_tests/test.xml + /bazel-testlogs/src/ext/score_.../score_any_folder_tests/test.xml Therefore will we 'clean' it before passing it to the parse_module func. Removing everything up to and including 'bazel-testlogs' or 'tests-report' """ - #print("THIs IS FILEPATH IN GET MD FROm TestPATH: ", raw_filepath) + # print("THIs IS FILEPATH IN GET MD FROm TestPATH: ", raw_filepath) known_good_json = os.environ.get("KNOWN_GOOD_JSON") clean_filepath = clean_test_file_name(raw_filepath) - #print(f"This is the cleaned filepath: {clean_filepath}") + # print(f"This is the cleaned filepath: {clean_filepath}") module_name = parse_module_name_from_path(clean_filepath) md: MetaData = { "module_name": module_name, From a836298041b982e130fad2b915b9fb60417a9266 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Tue, 17 Mar 2026 15:11:45 +0100 Subject: [PATCH 24/34] Make comment more visibile --- .../score_source_code_linker/tests/test_xml_parser.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/extensions/score_source_code_linker/tests/test_xml_parser.py b/src/extensions/score_source_code_linker/tests/test_xml_parser.py index b33000121..3c328fedf 100644 --- a/src/extensions/score_source_code_linker/tests/test_xml_parser.py +++ b/src/extensions/score_source_code_linker/tests/test_xml_parser.py @@ -10,7 +10,11 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -# Some portions generated by CoPilot + +# ╓ ╖ +# ║ Some portions generated by CoPilot ║ +# ╙ ╜ + """ Tests for the xml_parser.py file. Keep in mind that this is with the 'assertions' inside xml_parser disabled so far. From 133a44df7ae25aabae45c6c668c8ee639f4f3976 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Wed, 18 Mar 2026 10:42:32 +0100 Subject: [PATCH 25/34] Small cleanup --- scripts_bazel/merge_sourcelinks.py | 2 +- src/extensions/score_source_code_linker/__init__.py | 9 ++++++--- .../generate_source_code_links_json.py | 12 ++++++++---- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/scripts_bazel/merge_sourcelinks.py b/scripts_bazel/merge_sourcelinks.py index a6cc03e29..147c02d11 100644 --- a/scripts_bazel/merge_sourcelinks.py +++ b/scripts_bazel/merge_sourcelinks.py @@ -48,7 +48,7 @@ def main(): _ = parser.add_argument( "--known_good", required=True, - help="Optional path to a 'known good' JSON file (provided by Bazel).", + help="Path to a required 'known good' JSON file (provided by Bazel).", ) _ = parser.add_argument( "files", diff --git a/src/extensions/score_source_code_linker/__init__.py b/src/extensions/score_source_code_linker/__init__.py index d80754036..679dc0a81 100644 --- a/src/extensions/score_source_code_linker/__init__.py +++ b/src/extensions/score_source_code_linker/__init__.py @@ -256,9 +256,12 @@ def setup_module_linker(app: Sphinx, _: BuildEnvironment): grouped_cache = get_cache_filename( app.outdir, "score_module_grouped_scl_cache.json" ) - gruped_cache_exists = grouped_cache.exists() + grouped_cache_exists = grouped_cache.exists() # TODO this cache should be done via Bazel - if not gruped_cache_exists or not app.config.skip_rescanning_via_source_code_linker: + if ( + not grouped_cache_exists + or not app.config.skip_rescanning_via_source_code_linker + ): LOGGER.debug( "Did not find combined json 'score_module_grouped_scl_cache.json' " "in _build. Generating new one" @@ -291,7 +294,7 @@ def setup_once(app: Sphinx): register_combined_linker(app) register_module_linker(app) - # Priorty=515 to ensure it's called after the test code linker & combined connection + # Priority=515 to ensure it's called after the test linker & combined connection app.connect("env-updated", inject_links_into_needs, priority=525) diff --git a/src/extensions/score_source_code_linker/generate_source_code_links_json.py b/src/extensions/score_source_code_linker/generate_source_code_links_json.py index cf9457c82..7c48de9ee 100644 --- a/src/extensions/score_source_code_linker/generate_source_code_links_json.py +++ b/src/extensions/score_source_code_linker/generate_source_code_links_json.py @@ -130,10 +130,14 @@ def find_all_need_references(search_path: Path) -> list[NeedLink]: # Use os.walk to have better control over directory traversal for file in iterate_files_recursively(search_path): - print("Search_path: ", search_path) - print("File.name: ", file.name) - print("File: ", file) - references = _extract_references_from_file(search_path,Path(file), file) + LOGGER.debug( + f"Scanning file by the name of: {file.name} " + f"in path: {search_path} with the file being: {file}" + ) + # print("Search_path: ", search_path) + # print("File.name: ", file.name) + # print("File: ", file) + references = _extract_references_from_file(search_path, Path(file), file) all_need_references.extend(references) elapsed_time = os.times().elapsed - start_time From 748467da975fd6f75dd8f0078c7986676b2aaabb Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Fri, 20 Mar 2026 10:53:58 +0100 Subject: [PATCH 26/34] Renaming module => repo --- scripts_bazel/generate_sourcelinks_cli.py | 13 +- scripts_bazel/merge_sourcelinks.py | 18 +- src/extensions/score_source_code_linker/BUILD | 2 +- .../score_source_code_linker/__init__.py | 34 +-- .../score_source_code_linker/helpers.py | 20 +- .../score_source_code_linker/needlinks.py | 32 +-- ...e_source_links.py => repo_source_links.py} | 82 ++++--- .../score_source_code_linker/testlink.py | 24 +- .../tests/expected_codelink.json | 8 +- .../tests/expected_grouped.json | 18 +- ...rouped.json => expected_repo_grouped.json} | 20 +- .../tests/expected_testlink.json | 10 +- .../tests/test_codelink.py | 176 +++++++-------- .../tests/test_helpers.py | 97 ++++---- ...y => test_repo_source_link_integration.py} | 208 +++++++++--------- ...rce_links.py => test_repo_source_links.py} | 186 ++++++++-------- .../test_source_code_link_integration.py | 40 ++-- .../tests/test_testlink.py | 24 +- .../tests/test_xml_parser.py | 6 +- .../score_source_code_linker/xml_parser.py | 28 ++- 20 files changed, 517 insertions(+), 529 deletions(-) rename src/extensions/score_source_code_linker/{module_source_links.py => repo_source_links.py} (62%) rename src/extensions/score_source_code_linker/tests/{expected_module_grouped.json => expected_repo_grouped.json} (80%) rename src/extensions/score_source_code_linker/tests/{test_module_source_links_integration.py => test_repo_source_link_integration.py} (64%) rename src/extensions/score_source_code_linker/tests/{test_module_source_links.py => test_repo_source_links.py} (76%) diff --git a/scripts_bazel/generate_sourcelinks_cli.py b/scripts_bazel/generate_sourcelinks_cli.py index 89898d0ee..e5b134a45 100644 --- a/scripts_bazel/generate_sourcelinks_cli.py +++ b/scripts_bazel/generate_sourcelinks_cli.py @@ -25,9 +25,9 @@ from src.extensions.score_source_code_linker.generate_source_code_links_json import ( _extract_references_from_file, # pyright: ignore[reportPrivateUsage] TODO: move it out of the extension and into this script ) -from src.extensions.score_source_code_linker.helpers import parse_module_name_from_path +from src.extensions.score_source_code_linker.helpers import parse_repo_name_from_path from src.extensions.score_source_code_linker.needlinks import ( - MetaData, + DefaultMetaData, store_source_code_links_with_metadata_json, ) @@ -71,15 +71,12 @@ def main(): args = parser.parse_args() all_need_references = [] - metadata: MetaData = { - "module_name": "", - "hash": "", - "url": "", - } + + metadata = DefaultMetaData() metadata_set = False for file_path in args.files: if "known_good.json" not in str(file_path) and not metadata_set: - metadata["module_name"] = parse_module_name_from_path(file_path) + metadata["repo_name"] = parse_repo_name_from_path(file_path) metadata_set = True abs_file_path = file_path.resolve() assert abs_file_path.exists(), abs_file_path diff --git a/scripts_bazel/merge_sourcelinks.py b/scripts_bazel/merge_sourcelinks.py index 147c02d11..88499d11a 100644 --- a/scripts_bazel/merge_sourcelinks.py +++ b/scripts_bazel/merge_sourcelinks.py @@ -27,14 +27,6 @@ logger = logging.getLogger(__name__) -""" -if bazel-out/k8-fastbuild/bin/external/ in file_path => module is external -otherwise it's local -if local => module_name & hash == empty -if external => parse thing for module_name => look up known_good json for hash & url -""" - - def main(): parser = argparse.ArgumentParser( description="Merge multiple sourcelinks JSON files into one" @@ -68,21 +60,21 @@ def main(): if not data: continue metadata = data[0] - if not isinstance(metadata, dict) or "module_name" not in metadata: + if not isinstance(metadata, dict) or "repo_name" not in metadata: logger.warning( f"Unexpected schema in sourcelinks file '{json_file}': " "expected first element to be a metadata dict " - "with a 'module_name' key. " + "with a 'repo_name' key. " ) # As we can't deal with bad JSON structure we just skip it continue - if metadata["module_name"] and metadata["module_name"] != "local_module": + if metadata["repo_name"] and metadata["repo_name"] != "local_repo": hash, repo = parse_info_from_known_good( - known_good_json=args.known_good, module_name=metadata["module_name"] + known_good_json=args.known_good, repo_name=metadata["repo_name"] ) metadata["hash"] = hash metadata["url"] = repo - # In the case that 'metadata[module_name]' is 'local_module' + # In the case that 'metadata[repo_name]' is 'local_module' # hash & url are already existing and empty inside of 'metadata' # Therefore all 3 keys will be written to needlinks in each branch diff --git a/src/extensions/score_source_code_linker/BUILD b/src/extensions/score_source_code_linker/BUILD index 85b8f222b..e9e3528f7 100644 --- a/src/extensions/score_source_code_linker/BUILD +++ b/src/extensions/score_source_code_linker/BUILD @@ -55,7 +55,7 @@ py_library( "testlink.py", "xml_parser.py", "helpers.py", - "module_source_links.py", + "repo_source_links.py", ], imports = ["."], visibility = ["//visibility:public"], diff --git a/src/extensions/score_source_code_linker/__init__.py b/src/extensions/score_source_code_linker/__init__.py index 679dc0a81..59d336ceb 100644 --- a/src/extensions/score_source_code_linker/__init__.py +++ b/src/extensions/score_source_code_linker/__init__.py @@ -35,10 +35,10 @@ generate_source_code_links_json, ) from src.extensions.score_source_code_linker.helpers import get_github_link -from src.extensions.score_source_code_linker.module_source_links import ( - group_needs_by_module, - load_module_source_links_json, - store_module_source_links_json, +from src.extensions.score_source_code_linker.repo_source_links import ( + group_needs_by_repo, + load_repo_source_links_json, + store_repo_source_links_json, ) from src.extensions.score_source_code_linker.need_source_links import ( group_by_need, @@ -235,26 +235,26 @@ def setup_combined_linker(app: Sphinx, _: BuildEnvironment): build_and_save_combined_file(app.outdir) -def register_module_linker(app: Sphinx): +def register_repo_linker(app: Sphinx): # Registering the combined linker to Sphinx # priority is set to make sure it is called in the right order. # Needs to be called after xml parsing & codelink - app.connect("env-updated", setup_module_linker, priority=520) + app.connect("env-updated", setup_repo_linker, priority=520) -def build_and_save_module_scl_file(outdir: Path): +def build_and_save_repo_scl_file(outdir: Path): scl_links = load_source_code_links_combined_json( get_cache_filename(outdir, "score_scl_grouped_cache.json") ) - mcl_links = group_needs_by_module(scl_links) - store_module_source_links_json( - outdir / "score_module_grouped_scl_cache.json", mcl_links + mcl_links = group_needs_by_repo(scl_links) + store_repo_source_links_json( + outdir / "score_repo_grouped_scl_cache.json", mcl_links ) -def setup_module_linker(app: Sphinx, _: BuildEnvironment): +def setup_repo_linker(app: Sphinx, _: BuildEnvironment): grouped_cache = get_cache_filename( - app.outdir, "score_module_grouped_scl_cache.json" + app.outdir, "score_repo_grouped_scl_cache.json" ) grouped_cache_exists = grouped_cache.exists() # TODO this cache should be done via Bazel @@ -266,7 +266,7 @@ def setup_module_linker(app: Sphinx, _: BuildEnvironment): "Did not find combined json 'score_module_grouped_scl_cache.json' " "in _build. Generating new one" ) - build_and_save_module_scl_file(app.outdir) + build_and_save_repo_scl_file(app.outdir) def setup_once(app: Sphinx): @@ -292,7 +292,7 @@ def setup_once(app: Sphinx): setup_source_code_linker(app, ws_root) register_test_code_linker(app) register_combined_linker(app) - register_module_linker(app) + register_repo_linker(app) # Priority=515 to ensure it's called after the test linker & combined connection app.connect("env-updated", inject_links_into_needs, priority=525) @@ -350,8 +350,8 @@ def inject_links_into_needs(app: Sphinx, env: BuildEnvironment) -> None: f"?? Need {id} already has testlink: {need.get('testlink')}" ) - scl_by_module = load_module_source_links_json( - get_cache_filename(app.outdir, "score_module_grouped_scl_cache.json") + scl_by_module = load_repo_source_links_json( + get_cache_filename(app.outdir, "score_repo_grouped_scl_cache.json") ) for module_grouped_needs in scl_by_module: for source_code_links in module_grouped_needs.needs: @@ -373,7 +373,7 @@ def inject_links_into_needs(app: Sphinx, env: BuildEnvironment) -> None: continue need_as_dict = cast(dict[str, object], need) - metadata = module_grouped_needs.module + metadata = module_grouped_needs.repo need_as_dict["source_code_link"] = ", ".join( f"{get_github_link(metadata, n)}<>{n.file}:{n.line}" for n in source_code_links.links.CodeLinks diff --git a/src/extensions/score_source_code_linker/helpers.py b/src/extensions/score_source_code_linker/helpers.py index a186754ac..b41b7cc56 100644 --- a/src/extensions/score_source_code_linker/helpers.py +++ b/src/extensions/score_source_code_linker/helpers.py @@ -13,7 +13,7 @@ import json from pathlib import Path -from src.extensions.score_source_code_linker.module_source_links import ModuleInfo +from src.extensions.score_source_code_linker.repo_source_links import RepoInfo # Import types that depend on score_source_code_linker from src.extensions.score_source_code_linker.needlinks import DefaultNeedLink, NeedLink @@ -29,7 +29,7 @@ def get_github_link( - metadata: ModuleInfo, + metadata: RepoInfo, link: NeedLink | DataForTestLink | DataOfTestCase | None = None, ) -> str: if link is None: @@ -55,7 +55,7 @@ def get_github_link_from_git( def get_github_link_from_json( - metadata: ModuleInfo, + metadata: RepoInfo, link: NeedLink | DataForTestLink | DataOfTestCase | None = None, ) -> str: if link is None: @@ -65,7 +65,7 @@ def get_github_link_from_json( return f"{base_url}/blob/{current_hash}/{link.file}#L{link.line}" -def parse_module_name_from_path(path: Path) -> str: +def parse_repo_name_from_path(path: Path) -> str: """ Parse out the Module-Name from the filename: Combo Example: @@ -73,7 +73,7 @@ def parse_module_name_from_path(path: Path) -> str: => score_docs_as_code Local: Path: src/helper_lib/test_helper_lib.py - => local_module + => local_repo """ @@ -85,11 +85,11 @@ def parse_module_name_from_path(path: Path) -> str: filepath_split = str(module_raw).split("/", maxsplit=1) return str(filepath_split[0].removesuffix("+")) # We return this when we are in a local build `//:docs` the rest of DaC knows - return "local_module" + return "local_repo" def parse_info_from_known_good( - known_good_json: Path, module_name: str + known_good_json: Path, repo_name: str ) -> tuple[str, str]: with open(known_good_json) as f: kg_json = json.load(f) @@ -106,7 +106,7 @@ def parse_info_from_known_good( ) for category in kg_json["modules"].values(): - if module_name in category: - m = category[module_name] + if repo_name in category: + m = category[repo_name] return (m["hash"], m["repo"].removesuffix(".git")) - raise KeyError(f"Module {module_name!r} not found in known_good_json.") + raise KeyError(f"Module {repo_name} not found in known_good_json.") diff --git a/src/extensions/score_source_code_linker/needlinks.py b/src/extensions/score_source_code_linker/needlinks.py index 52f424830..fb099e14a 100644 --- a/src/extensions/score_source_code_linker/needlinks.py +++ b/src/extensions/score_source_code_linker/needlinks.py @@ -20,14 +20,19 @@ class MetaData(TypedDict): - module_name: str + repo_name: str hash: str url: str +def DefaultMetaData() -> MetaData: + md: MetaData = {"repo_name": "local_repo", "hash": "", "url": ""} + return md + + def is_metadata(x: object) -> TypeGuard[MetaData]: # Make this as strict/loose as you want; at minimum, it must be a dict. - return isinstance(x, dict) and {"module_name", "hash", "url"} <= x.keys() + return isinstance(x, dict) and {"repo_name", "hash", "url"} <= x.keys() @dataclass(order=True) @@ -39,12 +44,13 @@ class NeedLink: tag: str need: str full_line: str - module_name: str = "local_module" + repo_name: str = "local_repo" hash: str = "" url: str = "" # Adding hashing & equality as this is needed to make comparisions. # Since the Dataclass is not 'frozen = true' it isn't automatically hashable + # This is used in tests def __hash__(self): return hash( ( @@ -53,7 +59,7 @@ def __hash__(self): self.tag, self.need, self.full_line, - self.module_name, + self.repo_name, self.hash, self.url, ) @@ -68,7 +74,7 @@ def __eq__(self, other: Any): and self.tag == other.tag and self.need == other.need and self.full_line == other.full_line - and self.module_name == other.module_name + and self.repo_name == other.repo_name and self.hash == other.hash and self.url == other.url ) @@ -78,10 +84,10 @@ def to_dict_full(self) -> dict[str, str | Path | int]: return asdict(self) # Drops MetaData fields for saving the Dataclass (saving space in json) - # The information is in the 'Module_Source_Link' in the end + # The information is in the 'Repo_Source_Link' in the end def to_dict_without_metadata(self) -> dict[str, str | Path | int]: d = asdict(self) - d.pop("module_name", None) + d.pop("repo_name", None) d.pop("hash", None) d.pop("url", None) return d @@ -98,7 +104,7 @@ def DefaultNeedLink() -> NeedLink: tag="", need="", full_line="", - # Module_name, hash, url are defaulted to "" + # Repo_name, hash, url are defaulted to "" # therefore not needed to be listed ) @@ -120,7 +126,7 @@ def needlink_decoder(d: dict[str, Any]) -> NeedLink | dict[str, Any]: tag=d["tag"], need=d["need"], full_line=d["full_line"], - module_name=d.get("module_name", ""), + repo_name=d.get("repo_name", ""), hash=d.get("hash", ""), url=d.get("url", ""), ) @@ -136,13 +142,13 @@ def store_source_code_links_with_metadata_json( [ meta_dict, needlink1, needlink2, ... ] meta_dict must include: - module_name, hash, url + repo_name, hash, url """ payload: list[object] = [metadata, *needlist] # After `rm -rf _build` or on clean builds the directory does not exist, # so we need to create it. We create any folder that might be missing - file.parent.mkdir(exist_ok=True,parents=True) + file.parent.mkdir(exist_ok=True, parents=True) with open(file, "w", encoding="utf-8") as f: json.dump(payload, f, cls=NeedLinkEncoder, indent=2, ensure_ascii=False) @@ -197,7 +203,7 @@ def load_source_code_links_with_metadata_json(file: Path) -> list[NeedLink]: f"metadata must decode to NeedLink objects. File: {file}" ) for d in links: - d.module_name = metadata["module_name"] + d.repo_name = metadata["repo_name"] d.hash = metadata["hash"] d.url = metadata["url"] return links @@ -206,7 +212,7 @@ def load_source_code_links_with_metadata_json(file: Path) -> list[NeedLink]: def load_source_code_links_json(file: Path) -> list[NeedLink]: """ Expects the JSON array with needlinks - *that already have extra info in them* (module_name, hash, url): + *that already have extra info in them* (repo_name, hash, url): [ needlink1, needlink2, ... ] Returns: [NeedLink, NeedLink, ...] diff --git a/src/extensions/score_source_code_linker/module_source_links.py b/src/extensions/score_source_code_linker/repo_source_links.py similarity index 62% rename from src/extensions/score_source_code_linker/module_source_links.py rename to src/extensions/score_source_code_linker/repo_source_links.py index 1521f6868..841c20780 100644 --- a/src/extensions/score_source_code_linker/module_source_links.py +++ b/src/extensions/score_source_code_linker/repo_source_links.py @@ -27,19 +27,23 @@ @dataclass -class ModuleInfo: +class RepoInfo: name: str hash: str url: str +def DefaultRepoInfo() -> RepoInfo: + return RepoInfo(name="local_repo", hash="", url="") + + @dataclass -class ModuleSourceLinks: - module: ModuleInfo +class RepoSourceLinks: + repo: RepoInfo needs: list[SourceCodeLinks] = field(default_factory=list) -class ModuleSourceLinks_TEST_JSON_Encoder(json.JSONEncoder): +class RepoSourceLinks_TEST_JSON_Encoder(json.JSONEncoder): def default(self, o: object) -> str | dict[str, Any]: if isinstance(o, Path): return str(o) @@ -52,9 +56,9 @@ def default(self, o: object) -> str | dict[str, Any]: # dictionaries won't get split up and we will not # run into the 'to_dict_without_metadata' as # everything will be converted to a normal dictionary - if isinstance(o, ModuleSourceLinks): + if isinstance(o, RepoSourceLinks): return { - "module": asdict(o.module), + "repo": asdict(o.repo), "needs": o.needs, # Let the encoder handle the list } if isinstance(o, SourceCodeLinks): @@ -70,17 +74,17 @@ def default(self, o: object) -> str | dict[str, Any]: return super().default(o) -def ModuleSourceLinks_JSON_Decoder( +def RepoSourceLinks_JSON_Decoder( d: dict[str, Any], -) -> ModuleSourceLinks | dict[str, Any]: - if "module" in d and "needs" in d: - module = d["module"] +) -> RepoSourceLinks | dict[str, Any]: + if "repo" in d and "needs" in d: + repo = d["repo"] needs = d["needs"] - return ModuleSourceLinks( - module=ModuleInfo( - name=module.get("name"), - hash=module.get("hash"), - url=module.get("url"), + return RepoSourceLinks( + repo=RepoInfo( + name=repo.get("name"), + hash=repo.get("hash"), + url=repo.get("url"), ), # We know this can only be list[SourceCodeLinks] and nothing else # Therefore => we ignore the type error here @@ -89,8 +93,8 @@ def ModuleSourceLinks_JSON_Decoder( return d -def store_module_source_links_json( - file: Path, source_code_links: list[ModuleSourceLinks] +def store_repo_source_links_json( + file: Path, source_code_links: list[RepoSourceLinks] ): # After `rm -rf _build` or on clean builds the directory does not exist, # so we need to create it. We create any folder that might be missing @@ -99,31 +103,31 @@ def store_module_source_links_json( json.dump( source_code_links, f, - cls=ModuleSourceLinks_TEST_JSON_Encoder, + cls=RepoSourceLinks_TEST_JSON_Encoder, indent=2, ensure_ascii=False, ) -def load_module_source_links_json(file: Path) -> list[ModuleSourceLinks]: - links: list[ModuleSourceLinks] = json.loads( +def load_repo_source_links_json(file: Path) -> list[RepoSourceLinks]: + links: list[RepoSourceLinks] = json.loads( file.read_text(encoding="utf-8"), - object_hook=ModuleSourceLinks_JSON_Decoder, + object_hook=RepoSourceLinks_JSON_Decoder, ) assert isinstance(links, list), ( - "The ModuleSourceLink json should be a list of ModuleSourceLink objects." + "The RepoSourceLink json should be a list of RepoSourceLink objects." ) - assert all(isinstance(link, ModuleSourceLinks) for link in links), ( - "All items in module source link cache should be ModuleSourceLink objects." + assert all(isinstance(link, RepoSourceLinks) for link in links), ( + "All items in repo source link cache should be RepoSourceLink objects." ) return links -def group_needs_by_module(links: list[SourceCodeLinks]) -> list[ModuleSourceLinks]: - module_groups: dict[str, ModuleSourceLinks] = {} +def group_needs_by_repo(links: list[SourceCodeLinks]) -> list[RepoSourceLinks]: + repo_groups: dict[str, RepoSourceLinks] = {} for source_link in links: - # Check if we can take moduleInfo from code or testlinks + # Check if we can take repoInfo from code or testlinks if source_link.links.CodeLinks: first_link = source_link.links.CodeLinks[0] elif source_link.links.TestLinks: @@ -131,28 +135,22 @@ def group_needs_by_module(links: list[SourceCodeLinks]) -> list[ModuleSourceLink else: # This should not happen? continue - module_key = first_link.module_name + repo_key = first_link.repo_name - if module_key not in module_groups: - module_groups[module_key] = ModuleSourceLinks( - module=ModuleInfo( - name=module_key, hash=first_link.hash, url=first_link.url + if repo_key not in repo_groups: + repo_groups[repo_key] = RepoSourceLinks( + repo=RepoInfo( + name=repo_key, hash=first_link.hash, url=first_link.url ) ) # TODO: Add an assert that checks if needs only are - # in a singular module (not allowed to be in multiple) - module_groups[module_key].needs.append(source_link) + # in a singular repo (not allowed to be in multiple) + repo_groups[repo_key].needs.append(source_link) return [ - ModuleSourceLinks(module=group.module, needs=group.needs) - for group in module_groups.values() + RepoSourceLinks(repo=group.repo, needs=group.needs) + for group in repo_groups.values() ] -# # Pouplate Metadata -# # Since all metadata inside the Codelinks is the same -# # we can just arbitrarily grab the first one -# module_name=need_links.CodeLinks[0].module_name, -# hash=need_links.CodeLinks[0].hash, -# url=need_links.CodeLinks[0].url, diff --git a/src/extensions/score_source_code_linker/testlink.py b/src/extensions/score_source_code_linker/testlink.py index 0dc1557f3..32f5cca8a 100644 --- a/src/extensions/score_source_code_linker/testlink.py +++ b/src/extensions/score_source_code_linker/testlink.py @@ -42,7 +42,7 @@ class DataForTestLink: verify_type: str result: str result_text: str = "" - module_name: str = "local_module" + repo_name: str = "local_repo" hash: str = "" url: str = "" @@ -58,7 +58,7 @@ def __hash__(self): self.verify_type, self.result, self.result_text, - self.module_name, + self.repo_name, self.hash, self.url, ) @@ -75,7 +75,7 @@ def __eq__(self, other: Any): and self.verify_type == other.verify_type and self.result == other.result and self.result_text == other.result_text - and self.module_name == other.module_name + and self.repo_name == other.repo_name and self.hash == other.hash and self.url == other.url ) @@ -85,10 +85,10 @@ def to_dict_full(self) -> dict[str, str | Path | int]: return asdict(self) # Drops MetaData fields for saving the Dataclass (saving space in json) - # The information is in the 'Module_Source_Link' in the end + # The information is in the 'Repo_Source_Link' in the end def to_dict_without_metadata(self) -> dict[str, str | Path | int]: d = asdict(self) - d.pop("module_name", None) + d.pop("repo_name", None) d.pop("hash", None) d.pop("url", None) return d @@ -110,7 +110,7 @@ def DataForTestLink_JSON_Decoder(d: dict[str, Any]) -> DataForTestLink | dict[st "line", "need", "verify_type", - "module_name", + "repo_name", "hash", "url", "result", @@ -121,7 +121,7 @@ def DataForTestLink_JSON_Decoder(d: dict[str, Any]) -> DataForTestLink | dict[st file=Path(d["file"]), line=d["line"], need=d["need"], - module_name=d.get("module_name", ""), + repo_name=d.get("repo_name", ""), hash=d.get("hash", ""), url=d.get("url", ""), verify_type=d["verify_type"], @@ -139,7 +139,7 @@ class DataOfTestCase: file: str | None = None line: str | None = None result: str | None = None # passed | falied | skipped | disabled - module_name: str | None = None + repo_name: str | None = None hash: str | None = None url: str | None = None # Intentionally not snakecase to make dict parsing simple @@ -157,7 +157,7 @@ def from_dict(cls, data: dict[str, Any]): # type-ignore file=data.get("file"), line=data.get("line"), result=data.get("result"), - module_name=data.get("module_name"), + repo_name=data.get("repo_name"), hash=data.get("hash"), url=data.get("url"), TestType=data.get("TestType"), @@ -221,7 +221,7 @@ def is_valid(self) -> bool: # and self.DerivationTechnique is not None # ): # Hash & URL are explictily allowed to be empty but not none. - # module_name has to be always filled or something went wrong + # repo_name has to be always filled or something went wrong fields = [ x for x in self.__dataclass_fields__ @@ -263,7 +263,7 @@ def parse_attributes(verify_field: str | None, verify_type: str): assert self.file is not None assert self.line is not None assert self.result is not None - assert self.module_name is not None + assert self.repo_name is not None assert self.hash is not None assert self.url is not None assert self.result_text is not None @@ -279,7 +279,7 @@ def parse_attributes(verify_field: str | None, verify_type: str): verify_type=verify_type, result=self.result, result_text=self.result_text, - module_name=self.module_name, + repo_name=self.repo_name, hash=self.hash, url=self.url, ) diff --git a/src/extensions/score_source_code_linker/tests/expected_codelink.json b/src/extensions/score_source_code_linker/tests/expected_codelink.json index 4ff1f14bb..422e88352 100644 --- a/src/extensions/score_source_code_linker/tests/expected_codelink.json +++ b/src/extensions/score_source_code_linker/tests/expected_codelink.json @@ -5,7 +5,7 @@ "tag":"#-----req-Id:", "need": "TREQ_ID_1", "full_line": "#-----req-Id: TREQ_ID_1", - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "" }, @@ -15,7 +15,7 @@ "tag":"#-----req-Id:", "need": "TREQ_ID_1", "full_line": "#-----req-Id: TREQ_ID_1", - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "" }, @@ -25,7 +25,7 @@ "tag":"#-----req-Id:", "need": "TREQ_ID_2", "full_line":"#-----req-Id: TREQ_ID_2", - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "" }, @@ -35,7 +35,7 @@ "tag":"#-----req-Id:", "need": "TREQ_ID_200", "full_line":"#-----req-Id: TREQ_ID_200", - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "" } diff --git a/src/extensions/score_source_code_linker/tests/expected_grouped.json b/src/extensions/score_source_code_linker/tests/expected_grouped.json index bd6df7e40..e2668e476 100644 --- a/src/extensions/score_source_code_linker/tests/expected_grouped.json +++ b/src/extensions/score_source_code_linker/tests/expected_grouped.json @@ -9,7 +9,7 @@ "tag":"#-----req-Id:", "need": "TREQ_ID_1", "full_line": "#-----req-Id: TREQ_ID_1", - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "" }, @@ -19,7 +19,7 @@ "tag":"#-----req-Id:", "need": "TREQ_ID_1", "full_line": "#-----req-Id: TREQ_ID_1", - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "" } @@ -34,7 +34,7 @@ "verify_type": "fully", "result": "passed", "result_text": "", - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "" } @@ -51,7 +51,7 @@ "tag":"#-----req-Id:", "need": "TREQ_ID_2", "full_line":"#-----req-Id: TREQ_ID_2", - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "" } @@ -66,7 +66,7 @@ "verify_type": "partially", "result": "passed", "result_text": "", - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "" }, @@ -78,7 +78,7 @@ "verify_type": "partially", "result": "passed", "result_text": "", - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "" } @@ -99,7 +99,7 @@ "verify_type": "partially", "result": "passed", "result_text": "", - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "" }, @@ -111,7 +111,7 @@ "verify_type": "partially", "result": "passed", "result_text": "", - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "" } @@ -128,7 +128,7 @@ "tag":"#-----req-Id:", "need": "TREQ_ID_200", "full_line":"#-----req-Id: TREQ_ID_200", - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "" } diff --git a/src/extensions/score_source_code_linker/tests/expected_module_grouped.json b/src/extensions/score_source_code_linker/tests/expected_repo_grouped.json similarity index 80% rename from src/extensions/score_source_code_linker/tests/expected_module_grouped.json rename to src/extensions/score_source_code_linker/tests/expected_repo_grouped.json index ef91784bf..3d0523121 100644 --- a/src/extensions/score_source_code_linker/tests/expected_module_grouped.json +++ b/src/extensions/score_source_code_linker/tests/expected_repo_grouped.json @@ -1,7 +1,7 @@ [ { - "module": { - "name": "local_module", + "repo": { + "name": "local_repo", "hash": "", "url": "" }, @@ -11,14 +11,14 @@ "links": { "CodeLinks": [ { - "file": "src/module_a_impl.py", + "file": "src/repo_a_impl.py", "line": 3, "tag": "#-----req-Id:", "need": "MOD_REQ_1", "full_line": "#-----req-Id: MOD_REQ_1" }, { - "file": "src/module_b_impl.py", + "file": "src/repo_b_impl.py", "line": 3, "tag": "#-----req-Id:", "need": "MOD_REQ_1", @@ -27,8 +27,8 @@ ], "TestLinks": [ { - "name": "test_module_a", - "file": "src/test_module_a.py", + "name": "test_repo_a", + "file": "src/test_repo_a.py", "line": 10, "need": "MOD_REQ_1", "verify_type": "fully", @@ -43,7 +43,7 @@ "links": { "CodeLinks": [ { - "file": "src/module_a_impl.py", + "file": "src/repo_a_impl.py", "line": 7, "tag": "#-----req-Id:", "need": "MOD_REQ_2", @@ -58,7 +58,7 @@ "links": { "CodeLinks": [ { - "file": "src/module_b_impl.py", + "file": "src/repo_b_impl.py", "line": 7, "tag": "#-----req-Id:", "need": "MOD_REQ_3", @@ -67,8 +67,8 @@ ], "TestLinks": [ { - "name": "test_module_b", - "file": "src/test_module_b.py", + "name": "test_repo_b", + "file": "src/test_repo_b.py", "line": 20, "need": "MOD_REQ_3", "verify_type": "partially", diff --git a/src/extensions/score_source_code_linker/tests/expected_testlink.json b/src/extensions/score_source_code_linker/tests/expected_testlink.json index 145c31c31..23fbbe395 100644 --- a/src/extensions/score_source_code_linker/tests/expected_testlink.json +++ b/src/extensions/score_source_code_linker/tests/expected_testlink.json @@ -7,7 +7,7 @@ "verify_type": "partially", "result": "passed", "result_text": "", - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "" }, @@ -19,7 +19,7 @@ "verify_type": "partially", "result": "passed", "result_text": "", - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "" }, @@ -31,7 +31,7 @@ "verify_type": "partially", "result": "passed", "result_text": "", - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "" }, @@ -43,7 +43,7 @@ "verify_type": "partially", "result": "passed", "result_text": "", - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "" }, @@ -55,7 +55,7 @@ "verify_type": "fully", "result": "passed", "result_text": "", - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "" } diff --git a/src/extensions/score_source_code_linker/tests/test_codelink.py b/src/extensions/score_source_code_linker/tests/test_codelink.py index 1328e1fa2..25387e7ec 100644 --- a/src/extensions/score_source_code_linker/tests/test_codelink.py +++ b/src/extensions/score_source_code_linker/tests/test_codelink.py @@ -23,6 +23,8 @@ from dataclasses import asdict from pathlib import Path from typing import Any +from collections import Counter +from collections.abc import Callable import pytest @@ -43,7 +45,7 @@ from src.extensions.score_source_code_linker.helpers import ( get_github_link, ) -from src.extensions.score_source_code_linker.module_source_links import ModuleInfo +from src.extensions.score_source_code_linker.repo_source_links import RepoInfo from src.extensions.score_source_code_linker.needlinks import ( MetaData, NeedLink, @@ -137,7 +139,7 @@ def needlink_test_decoder(d: dict[str, Any]) -> NeedLink | dict[str, Any]: "tag", "need", "full_line", - "module_name", + "repo_name", "hash", "url", } <= d.keys(): @@ -147,7 +149,7 @@ def needlink_test_decoder(d: dict[str, Any]) -> NeedLink | dict[str, Any]: tag=decode_comment(d["tag"]), need=d["need"], full_line=decode_comment(d["full_line"]), - module_name=d.get("module_name", ""), + repo_name=d.get("repo_name", ""), hash=d.get("hash", ""), url=d.get("url", ""), ) @@ -200,7 +202,7 @@ def sample_needlinks() -> list[NeedLink]: tag="#" + " req-Id:", need="TREQ_ID_1", full_line="#" + " req-Id: TREQ_ID_1", - module_name="local_module", + repo_name="local_repo", hash="", url="", ), @@ -210,7 +212,7 @@ def sample_needlinks() -> list[NeedLink]: tag="#" + " req-Id:", need="TREQ_ID_1", full_line="#" + " req-Id: TREQ_ID_1", - module_name="local_module", + repo_name="local_repo", hash="", url="", ), @@ -220,7 +222,7 @@ def sample_needlinks() -> list[NeedLink]: tag="#" + " req-Id:", need="TREQ_ID_2", full_line="#" + " req-Id: TREQ_ID_2", - module_name="local_module", + repo_name="local_repo", hash="", url="", ), @@ -230,7 +232,7 @@ def sample_needlinks() -> list[NeedLink]: tag="#" + " req-Id:", need="TREQ_ID_200", full_line="#" + " req-Id: TREQ_ID_200", - module_name="local_module", + repo_name="local_repo", hash="", url="", ), @@ -368,7 +370,7 @@ def test_cache_file_with_encoded_comments(temp_dir: Path) -> None: tag="#" + " req-Id:", need="TEST_001", full_line="#" + " req-Id: TEST_001", - module_name="local_module", + repo_name="local_repo", hash="", url="", ) @@ -462,7 +464,7 @@ def another_function(): tag="#" + " req-Id:", need="TREQ_ID_1", full_line="#" + " req-Id: TREQ_ID_1", - module_name="local_module", + repo_name="local_repo", hash="", url="", ), @@ -472,7 +474,7 @@ def another_function(): tag="#" + " req-Id:", need="TREQ_ID_2", full_line="#" + " req-Id: TREQ_ID_2", - module_name="local_module", + repo_name="local_repo", hash="", url="", ), @@ -482,7 +484,7 @@ def another_function(): tag="#" + " req-Id:", need="TREQ_ID_1", full_line="#" + " req-Id: TREQ_ID_1", - module_name="local_module", + repo_name="local_repo", hash="", url="", ), @@ -508,7 +510,7 @@ def another_function(): # Test GitHub link generation # Have to change directories in order to ensure that we get the right/any .git file os.chdir(Path(git_repo).absolute()) - metadata = ModuleInfo(name="local_module", hash="", url="") + metadata = RepoInfo(name="local_repo", hash="", url="") for needlink in loaded_links: github_link = get_github_link(metadata, needlink) assert "https://github.com/test-user/test-repo/blob/" in github_link @@ -544,7 +546,7 @@ def test_multiple_commits_hash_consistency(git_repo: Path) -> None: full_line="#" + " req-Id: TREQ_ID_1", ) - metadata = ModuleInfo(name="local_module", hash="", url="") + metadata = RepoInfo(name="local_repo", hash="", url="") os.chdir(Path(git_repo).absolute()) github_link = get_github_link(metadata, needlink) assert new_hash in github_link @@ -552,7 +554,7 @@ def test_multiple_commits_hash_consistency(git_repo: Path) -> None: def test_is_metadata_missing_keys(): """Bad path: Dict missing required keys returns False""" - incomplete = {"module_name": "test", "hash": "abc"} + incomplete = {"repo_name": "test", "hash": "abc"} assert is_metadata(incomplete) is False @@ -566,14 +568,14 @@ def test_needlink_to_dict_without_metadata(): line=10, tag="#" + " req-Id:", need="REQ_1", - full_line="#"+" req-Id: REQ_1", - module_name="test_module", + full_line="#" + " req-Id: REQ_1", + repo_name="test_repo", url="https://github.com/test/repo", hash="abc123", ) result = needlink.to_dict_without_metadata() - assert "module_name" not in result + assert "repo_name" not in result assert "hash" not in result assert "url" not in result assert result["need"] == "REQ_1" @@ -588,13 +590,13 @@ def test_needlink_to_dict_full(): tag="#" + " req-Id:", need="REQ_1", full_line="#" + " req-Id: REQ_1", - module_name="test_module", + repo_name="test_repo", url="https://github.com/test/repo", hash="abc123", ) result = needlink.to_dict_full() - assert result["module_name"] == "test_module" + assert result["repo_name"] == "test_repo" assert result["hash"] == "abc123" assert result["url"] == "https://github.com/test/repo" assert result["need"] == "REQ_1" @@ -612,14 +614,14 @@ def test_needlink_encoder_includes_metadata(): tag="#" + " req-Id:", need="REQ_1", full_line="#" + " req-Id: REQ_1", - module_name="test_module", + repo_name="test_repo", url="https://github.com/test/repo", hash="abc123", ) result = encoder.default(needlink) assert isinstance(result, dict) - assert result["module_name"] == "test_module" + assert result["repo_name"] == "test_repo" assert result["hash"] == "abc123" assert result["url"] == "https://github.com/test/repo" @@ -634,10 +636,10 @@ def test_needlink_decoder_with_all_fields(): json_data: dict[str, Any] = { "file": "src/test.py", "line": 10, - "tag":"#" + " req-Id:", + "tag": "#" + " req-Id:", "need": "REQ_1", - "full_line": "#" +" req-Id: REQ_1", - "module_name": "test_module", + "full_line": "#" + " req-Id: REQ_1", + "repo_name": "test_repo", "hash": "abc123", "url": "https://github.com/test/repo", } @@ -646,7 +648,7 @@ def test_needlink_decoder_with_all_fields(): assert isinstance(result, NeedLink) assert result.file == Path("src/test.py") assert result.line == 10 - assert result.module_name == "test_module" + assert result.repo_name == "test_repo" def test_needlink_decoder_non_needlink_dict(): @@ -659,30 +661,6 @@ def test_needlink_decoder_non_needlink_dict(): # ───────────────[ Testing Encoding / Decoding ]───────────── -def test_store_and_load_source_code_links(tmp_path: Path): - """Happy path: Store and load without metadata""" - needlinks = [ - NeedLink( - file=Path("src/impl.py"), - line=42, - tag="#" + " req-Id:", - need="REQ_1", - full_line="#" + " req-Id: REQ_1", - module_name="mod", - url="url", - hash="hash", - ) - ] - - test_file = tmp_path / "standard.json" - store_source_code_links_json(test_file, needlinks) - loaded = load_source_code_links_json(test_file) - - assert len(loaded) == 1 - assert loaded[0].need == "REQ_1" - assert loaded[0].module_name == "mod" - - def test_load_validates_list_type(tmp_path: Path): """Bad path: Non-list JSON fails validation""" test_file = tmp_path / "not_list.json" @@ -707,7 +685,7 @@ def test_load_validates_all_items_are_needlinks(tmp_path: Path): def test_store_and_load_with_metadata(tmp_path: Path): """Happy path: Store with metadata dict and load correctly""" metadata: MetaData = { - "module_name": "external_module", + "repo_name": "external_repo", "hash": "commit_xyz", "url": "https://github.com/external/repo", } @@ -717,8 +695,8 @@ def test_store_and_load_with_metadata(tmp_path: Path): line=10, tag="#" + " req-Id:", need="REQ_1", - full_line="#"+" req-Id: REQ_1", - module_name="", # Will be filled from metadata + full_line="#" + " req-Id: REQ_1", + repo_name="", # Will be filled from metadata url="", hash="", ), @@ -727,8 +705,8 @@ def test_store_and_load_with_metadata(tmp_path: Path): line=20, tag="#" + " req-Id:", need="REQ_2", - full_line="#"+" req-Id: REQ_2", - module_name="", + full_line="#" + " req-Id: REQ_2", + repo_name="", url="", hash="", ), @@ -740,14 +718,17 @@ def test_store_and_load_with_metadata(tmp_path: Path): assert len(loaded) == 2 # Verify metadata was applied to all links - assert loaded[0].module_name == "external_module" - assert loaded[0].hash == "commit_xyz" - assert loaded[0].url == "https://github.com/external/repo" - assert loaded[1].module_name == "external_module" + for nl in loaded: + assert nl.repo_name == "external_repo" + assert nl.url == "https://github.com/external/repo" + assert nl.hash == "commit_xyz" def test_load_with_metadata_missing_metadata_dict(tmp_path: Path): - """Bad path: Loading file without metadata dict raises TypeError""" + """ + Test if loading file without metadata dict via + the scl_with_metadata loader raises TypeError + """ # Store without metadata (just needlinks) needlinks = [ NeedLink( @@ -755,7 +736,7 @@ def test_load_with_metadata_missing_metadata_dict(tmp_path: Path): line=1, tag="#" + " req-Id:", need="REQ_1", - full_line="#" +" req-Id: REQ_1", + full_line="#" + " req-Id: REQ_1", ) ] @@ -768,12 +749,12 @@ def test_load_with_metadata_missing_metadata_dict(tmp_path: Path): def test_load_with_metadata_invalid_items_after_metadata(tmp_path: Path): - """Bad path: Items after metadata dict are not NeedLinks""" + """Test wrong JSON structure""" test_file = tmp_path / "bad_items.json" # Manually create invalid JSON _ = test_file.write_text( json.dumps( - [{"module_name": "mod", "hash": "h", "url": "u"}, {"invalid": "structure"}] + [{"repo_name": "mod", "hash": "h", "url": "u"}, {"invalid": "structure"}] ) ) @@ -787,7 +768,7 @@ def test_load_with_metadata_invalid_items_after_metadata(tmp_path: Path): def test_load_resolves_relative_path_with_env_var( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ): - """Edge case: Relative path is resolved using BUILD_WORKSPACE_DIRECTORY""" + """Test if relative path is resolved using BUILD_WORKSPACE_DIRECTORY""" workspace = tmp_path / "workspace" workspace.mkdir() @@ -795,9 +776,9 @@ def test_load_resolves_relative_path_with_env_var( NeedLink( file=Path("src/test.py"), line=1, - tag="#"+" req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="#"+" req-Id: REQ_1", + full_line="#" + " req-Id: REQ_1", ) ] @@ -821,7 +802,7 @@ def test_load_with_metadata_resolves_relative_path( workspace.mkdir() metadata: MetaData = { - "module_name": "mod", + "repo_name": "mod", "hash": "h", "url": "u", } @@ -829,9 +810,9 @@ def test_load_with_metadata_resolves_relative_path( NeedLink( file=Path("src/test.py"), line=1, - tag="#"+" req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="#"+" req-Id: REQ_1", + full_line="#" + " req-Id: REQ_1", ) ] @@ -842,7 +823,7 @@ def test_load_with_metadata_resolves_relative_path( loaded = load_source_code_links_with_metadata_json(Path("metadata_cache.json")) assert len(loaded) == 1 - assert loaded[0].module_name == "mod" + assert loaded[0].repo_name == "mod" # ─────────────────────[ Roundtrip Tests ]─────────────────── @@ -854,20 +835,20 @@ def test_roundtrip_standard_format(tmp_path: Path): NeedLink( file=Path("src/file1.py"), line=10, - tag="#"+" req-Id:", + tag="#" + " req-Id:", need="REQ_A", - full_line="#"+" req-Id: REQ_A", - module_name="mod_a", + full_line="#" + " req-Id: REQ_A", + repo_name="mod_a", url="url_a", hash="hash_a", ), NeedLink( file=Path("src/file2.py"), line=20, - tag="#"+" req-Id:", + tag="#" + " req-Id:", need="REQ_B", - full_line="#"+" req-Id: REQ_B", - module_name="mod_b", + full_line="#" + " req-Id: REQ_B", + repo_name="mod_b", url="url_b", hash="hash_b", ), @@ -878,14 +859,15 @@ def test_roundtrip_standard_format(tmp_path: Path): loaded = load_source_code_links_json(test_file) assert len(loaded) == 2 - assert loaded[0].module_name == "mod_a" - assert loaded[1].module_name == "mod_b" + assert needlinks == loaded + assert loaded[0].repo_name == "mod_a" + assert loaded[1].repo_name == "mod_b" def test_roundtrip_metadata_format_applies_metadata(tmp_path: Path): """Happy path: Metadata format applies metadata to all links""" metadata: MetaData = { - "module_name": "shared_module", + "repo_name": "shared_repo", "hash": "shared_hash", "url": "https://github.com/shared/repo", } @@ -893,16 +875,16 @@ def test_roundtrip_metadata_format_applies_metadata(tmp_path: Path): NeedLink( file=Path("src/f1.py"), line=5, - tag="#"+" req-Id:", + tag="#" + " req-Id:", need="R1", - full_line="#"+" req-Id: R1", + full_line="#" + " req-Id: R1", ), NeedLink( file=Path("src/f2.py"), line=15, - tag="#"+" req-Id:", + tag="#" + " req-Id:", need="R2", - full_line="#"+" req-Id: R2", + full_line="#" + " req-Id: R2", ), ] @@ -913,7 +895,7 @@ def test_roundtrip_metadata_format_applies_metadata(tmp_path: Path): assert len(loaded) == 2 # Both should have the same metadata applied for link in loaded: - assert link.module_name == "shared_module" + assert link.repo_name == "shared_repo" assert link.hash == "shared_hash" assert link.url == "https://github.com/shared/repo" @@ -933,16 +915,16 @@ def test_roundtrip_empty_lists(tmp_path: Path): def test_json_format_with_metadata_has_separate_dict(tmp_path: Path): """Edge case: Verify metadata format has metadata as first element""" metadata: MetaData = { - "module_name": "test_mod", + "repo_name": "test_mod", "hash": "test_hash", "url": "test_url", } needlink = NeedLink( file=Path("src/test.py"), line=1, - tag="#"+" req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="#"+" req-Id: REQ_1", + full_line="#" + " req-Id: REQ_1", ) test_file = tmp_path / "metadata_format.json" @@ -954,7 +936,7 @@ def test_json_format_with_metadata_has_separate_dict(tmp_path: Path): assert isinstance(raw_json, list) assert len(raw_json) == 2 # metadata + 1 needlink # First element should be metadata dict - assert raw_json[0]["module_name"] == "test_mod" + assert raw_json[0]["repo_name"] == "test_mod" assert raw_json[0]["hash"] == "test_hash" assert raw_json[0]["url"] == "test_url" @@ -967,20 +949,20 @@ def test_needlink_equality_same_values(): link1 = NeedLink( file=Path("src/test.py"), line=10, - tag="#"+" req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="#"+" req-Id: REQ_1", - module_name="mod", + full_line="#" + " req-Id: REQ_1", + repo_name="mod", url="url", hash="hash", ) link2 = NeedLink( file=Path("src/test.py"), line=10, - tag="#"+" req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="#"+" req-Id: REQ_1", - module_name="mod", + full_line="#" + " req-Id: REQ_1", + repo_name="mod", url="url", hash="hash", ) @@ -994,16 +976,16 @@ def test_needlink_inequality_different_values(): link1 = NeedLink( file=Path("src/test.py"), line=10, - tag="#"+" req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="#"+" req-Id: REQ_1", + full_line="#" + " req-Id: REQ_1", ) link2 = NeedLink( file=Path("src/test.py"), line=20, # Different line - tag="#"+" req-Id:", + tag="#" + " req-Id:", need="REQ_1", - full_line="#"+" req-Id: REQ_1", + full_line="#" + " req-Id: REQ_1", ) assert link1 != link2 diff --git a/src/extensions/score_source_code_linker/tests/test_helpers.py b/src/extensions/score_source_code_linker/tests/test_helpers.py index 9ad02bc39..d55e4517b 100644 --- a/src/extensions/score_source_code_linker/tests/test_helpers.py +++ b/src/extensions/score_source_code_linker/tests/test_helpers.py @@ -23,66 +23,66 @@ get_github_link, get_github_link_from_json, parse_info_from_known_good, - parse_module_name_from_path, + parse_repo_name_from_path, ) -from src.extensions.score_source_code_linker.module_source_links import ModuleInfo +from src.extensions.score_source_code_linker.repo_source_links import RepoInfo from src.extensions.score_source_code_linker.needlinks import DefaultNeedLink from src.helper_lib import get_current_git_hash # ╭──────────────────────────────────────────────────────────╮ -# │ Tests for parse_module_name_from_path │ +# │ Tests for parse_repo_name_from_path │ # ╰──────────────────────────────────────────────────────────╯ -def test_parse_module_name_from_external_path(): - """Test parsing module name from external/combo build path.""" +def test_parse_repo_name_from_external_path(): + """Test parsing repo name from external/combo build path.""" path = Path("external/score_docs_as_code+/src/helper_lib/test_helper_lib.py") - result = parse_module_name_from_path(path) + result = parse_repo_name_from_path(path) assert result == "score_docs_as_code" -def test_parse_module_name_from_external_path_2(): +def test_parse_repo_name_from_external_path_2(): """Test that an extra 'external' in the path does not change the output""" path = Path( "external/score_docs_as_code+/src/external/helper_lib/test_helper_lib.py" ) - result = parse_module_name_from_path(path) + result = parse_repo_name_from_path(path) assert result == "score_docs_as_code" -def test_parse_module_name_from_local_path(): - """Test parsing module name from local build path.""" +def test_parse_repo_name_from_local_path(): + """Test parsing repo name from local build path.""" path = Path("src/helper_lib/test_helper_lib.py") - result = parse_module_name_from_path(path) + result = parse_repo_name_from_path(path) - assert result == "local_module" + assert result == "local_repo" -def test_parse_module_name_from_empty_path(): - """Test parsing module name from an empty path.""" +def test_parse_repo_name_from_empty_path(): + """Test parsing repo name from an empty path.""" path = Path("") - result = parse_module_name_from_path(path) + result = parse_repo_name_from_path(path) - assert result == "local_module" + assert result == "local_repo" -def test_parse_module_name_without_plus_suffix(): +def test_parse_repo_name_without_plus_suffix(): """ - Test parsing external module without plus suffix. + Test parsing external repo without plus suffix. This should never happen (due to bazel adding the '+') But testing this edge case anyway """ - path = Path("external/module_without_plus/file.py") + path = Path("external/repo_without_plus/file.py") - result = parse_module_name_from_path(path) + result = parse_repo_name_from_path(path) - assert result == "module_without_plus" + assert result == "repo_without_plus" # ╭──────────────────────────────────────────────────────────╮ @@ -126,7 +126,7 @@ def known_good_json(tmp_path: Path): # Tests for parse_info_from_known_good def test_parse_info_from_known_good_happy_path(known_good_json: Path): - """Test parsing module info from valid known_good.json.""" + """Test parsing repo info from valid known_good.json.""" hash_result, repo_result = parse_info_from_known_good( known_good_json, "score_baselibs" ) @@ -136,7 +136,7 @@ def test_parse_info_from_known_good_happy_path(known_good_json: Path): def test_parse_info_from_known_good_different_category(known_good_json: Path): - """Test finding module in different category.""" + """Test finding repo in different category.""" hash_result, repo_result = parse_info_from_known_good( known_good_json, "score_docs_as_code" ) @@ -145,14 +145,14 @@ def test_parse_info_from_known_good_different_category(known_good_json: Path): assert repo_result == "https://github.com/eclipse-score/docs-as-code" -def test_parse_info_from_known_good_module_not_found(known_good_json: Path): - """Test that KeyError is raised when module doesn't exist.""" - with pytest.raises(KeyError, match="Module 'nonexistent' not found"): +def test_parse_info_from_known_good_repo_not_found(known_good_json: Path): + """Test that KeyError is raised when repo doesn't exist.""" + with pytest.raises(KeyError, match="Module nonexistent not found"): parse_info_from_known_good(known_good_json, "nonexistent") def test_parse_info_from_known_good_empty_json(tmp_path: Path): - """Test with empty JSON file.""" + """Test that it errors when given an empty JSON file.""" json_file = tmp_path / "example.json" _ = json_file.write_text("{}") @@ -160,11 +160,11 @@ def test_parse_info_from_known_good_empty_json(tmp_path: Path): AssertionError, match=f"Known good json at: {json_file} is empty. This is not allowed", ): - parse_info_from_known_good(json_file, "any_module") + parse_info_from_known_good(json_file, "any_repo") -def test_parse_info_from_known_good_no_module_in_json(tmp_path: Path): - """Test that assertion works when module not in top level keys""" +def test_parse_info_from_known_good_no_repo_in_json(tmp_path: Path): + """Test that assertion works when repo not in top level keys""" json_file = tmp_path / "example.json" _ = json_file.write_text( '{"another_key": {"a": "b"}, "second_key": ["a" , "b", "c"]}' @@ -174,11 +174,11 @@ def test_parse_info_from_known_good_no_module_in_json(tmp_path: Path): AssertionError, match=f"Known good json at: {json_file} is missing the 'modules' key", ): - parse_info_from_known_good(json_file, "any_module") + parse_info_from_known_good(json_file, "any_repo") -def test_parse_info_from_known_good_empty_module_dict_in_json(tmp_path: Path): - """Test that assertion works if module dictionary is empty""" +def test_parse_info_from_known_good_empty_repo_dict_in_json(tmp_path: Path): + """Test that assertion works if repo dictionary is empty""" json_file = tmp_path / "emample.json" _ = json_file.write_text('{"another_key": {"a": "b"}, "modules": {}}') @@ -186,13 +186,16 @@ def test_parse_info_from_known_good_empty_module_dict_in_json(tmp_path: Path): AssertionError, match=f"Known good json at: {json_file} has an empty 'modules' dictionary", ): - parse_info_from_known_good(json_file, "any_module") + parse_info_from_known_good(json_file, "any_repo") # Tests for get_github_link_from_json def test_get_github_link_from_json_happy_path(): - """Test generating GitHub link from metadata.""" - metadata = ModuleInfo( + """ + Test generating GitHub link with fully filled metadata + =>this happens in combo builds + """ + metadata = RepoInfo( name="project_name", url="https://github.com/eclipse/project", hash="commit123abc", @@ -212,7 +215,7 @@ def test_get_github_link_from_json_happy_path(): def test_get_github_link_from_json_with_none_link(): """Test that None link creates DefaultNeedLink.""" - metadata = ModuleInfo( + metadata = RepoInfo( name="test_repo", url="https://github.com/test/repo", hash="def456" ) @@ -224,7 +227,7 @@ def test_get_github_link_from_json_with_none_link(): def test_get_github_link_from_json_with_line_zero(): """Test generating link with line number 0.""" - metadata = ModuleInfo( + metadata = RepoInfo( name="test_repo", url="https://github.com/test/repo", hash="hash123" ) @@ -240,8 +243,8 @@ def test_get_github_link_from_json_with_line_zero(): # Tests for get_github_link def test_get_github_link_with_hash(): """Test get_github_link uses json method when hash is present.""" - metadata = ModuleInfo( - name="some_module", url="https://github.com/org/repo", hash="commit_hash_123" + metadata = RepoInfo( + name="some_repo", url="https://github.com/org/repo", hash="commit_hash_123" ) link = DefaultNeedLink() @@ -254,12 +257,14 @@ def test_get_github_link_with_hash(): result == "https://github.com/org/repo/blob/commit_hash_123/src/example.py#L42" ) + @pytest.fixture def temp_dir() -> Generator[Path, None, None]: """Create a temporary directory for tests.""" with tempfile.TemporaryDirectory() as temp_dir: yield Path(temp_dir) + @pytest.fixture def git_repo(temp_dir: Path) -> Path: """Create a real git repository for testing.""" @@ -295,9 +300,9 @@ def git_repo(temp_dir: Path) -> Path: def test_get_github_link_with_real_repo(git_repo: Path) -> None: """ Test generating GitHub link without url/hash. - This expects to be in a git module + This expects to be in a git repo """ - metadata = ModuleInfo(name="some_module", url="", hash="") + metadata = RepoInfo(name="some_repo", url="", hash="") link = DefaultNeedLink() link.file = Path("src/example.py") @@ -323,16 +328,16 @@ def test_complete_workflow(known_good_json: Path): # Parse path path = Path("external/score_docs_as_code+/src/helper_lib/test_helper_lib.py") - module_name = parse_module_name_from_path(path) - assert module_name == "score_docs_as_code" + repo_name = parse_repo_name_from_path(path) + assert repo_name == "score_docs_as_code" # Get metadata - hash_val, repo_url = parse_info_from_known_good(known_good_json, module_name) + hash_val, repo_url = parse_info_from_known_good(known_good_json, repo_name) assert hash_val == "c1207676afe6cafd25c35d420e73279a799515d8" assert repo_url == "https://github.com/eclipse-score/docs-as-code" # Generate link - metadata = ModuleInfo(name=module_name, url=repo_url, hash=hash_val) + metadata = RepoInfo(name=repo_name, url=repo_url, hash=hash_val) link = DefaultNeedLink() link.file = Path("src/helper_lib/test_helper_lib.py") link.line = 75 diff --git a/src/extensions/score_source_code_linker/tests/test_module_source_links_integration.py b/src/extensions/score_source_code_linker/tests/test_repo_source_link_integration.py similarity index 64% rename from src/extensions/score_source_code_linker/tests/test_module_source_links_integration.py rename to src/extensions/score_source_code_linker/tests/test_repo_source_link_integration.py index 895a8eb56..270fef076 100644 --- a/src/extensions/score_source_code_linker/tests/test_module_source_links_integration.py +++ b/src/extensions/score_source_code_linker/tests/test_repo_source_link_integration.py @@ -22,10 +22,10 @@ from pytest import TempPathFactory from sphinx.testing.util import SphinxTestApp -from src.extensions.score_source_code_linker.module_source_links import ( - ModuleInfo, - ModuleSourceLinks_JSON_Decoder, - load_module_source_links_json, +from src.extensions.score_source_code_linker.repo_source_links import ( + RepoInfo, + RepoSourceLinks_JSON_Decoder, + load_repo_source_links_json, ) """ @@ -73,8 +73,8 @@ def create_demo_files(sphinx_base_dir: Path, git_repo_setup: Path): source_dir = repo_path / "src" source_dir.mkdir() - _ = (source_dir / "module_a_impl.py").write_text(make_module_a_source()) - _ = (source_dir / "module_b_impl.py").write_text(make_module_b_source()) + _ = (source_dir / "repo_a_impl.py").write_text(make_repo_a_source()) + _ = (source_dir / "repo_b_impl.py").write_text(make_repo_b_source()) # Create docs directory docs_dir = repo_path / "docs" @@ -89,8 +89,8 @@ def create_demo_files(sphinx_base_dir: Path, git_repo_setup: Path): curr_dir = Path(__file__).absolute().parent _ = shutil.copyfile( - curr_dir / "expected_module_grouped.json", - repo_path / ".expected_module_grouped.json", + curr_dir / "expected_repo_grouped.json", + repo_path / ".expected_repo_grouped.json", ) # Commit everything @@ -102,45 +102,53 @@ def create_demo_files(sphinx_base_dir: Path, git_repo_setup: Path): ) -def make_module_a_source(): - return """ -# Module A implementation -# """+"""req-Id: MOD_REQ_1 -def module_a_function(): +def make_repo_a_source(): + return ( + """ +# Repo A implementation +# """ + + """req-Id: MOD_REQ_1 +def repo_a_function(): pass -# """+"""req-Id: MOD_REQ_2 -class ModuleAClass: +# """ + + """req-Id: MOD_REQ_2 +class RepoAClass: pass """ + ) -def make_module_b_source(): - return """ -# Module B implementation -# """+"""req-Id: MOD_REQ_1 -def module_b_function(): +def make_repo_b_source(): + return ( + """ +# Repo B implementation +# """ + + """req-Id: MOD_REQ_1 +def repo_b_function(): pass -# """+"""req-Id: MOD_REQ_3 -def another_module_b_function(): +# """ + + """req-Id: MOD_REQ_3 +def another_repo_b_function(): pass """ + ) def make_test_xml(): # ruff: noqa: E501 (start) return """ - - + + - + @@ -162,7 +170,7 @@ def basic_conf(): needs_types = [ dict( directive="mod_req", - title="Module Requirement", + title="Repo Requirement", prefix="MOD_REQ_", color="#BFD8D2", style="node", @@ -187,15 +195,15 @@ def basic_needs(): MODULE TESTING ============== -.. mod_req:: Module Requirement 1 +.. mod_req:: Repo Requirement 1 :id: MOD_REQ_1 :status: valid -.. mod_req:: Module Requirement 2 +.. mod_req:: Repo Requirement 2 :id: MOD_REQ_2 :status: open -.. mod_req:: Module Requirement 3 +.. mod_req:: Repo Requirement 3 :id: MOD_REQ_3 :status: open """ @@ -231,89 +239,89 @@ def _create_app(): return _create_app -def test_module_grouped_cache_generated( +def test_repo_grouped_cache_generated( sphinx_app_setup: Callable[[], SphinxTestApp], sphinx_base_dir: Path, git_repo_setup: Path, create_demo_files: None, ): - """Happy path: Module grouped cache file is generated after Sphinx build""" + """Happy path: Repo grouped cache file is generated after Sphinx build""" app = sphinx_app_setup() try: os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) app.build() - module_cache = app.outdir / "score_module_grouped_scl_cache.json" - assert module_cache.exists(), "Module grouped cache was not created" + repo_cache = app.outdir / "score_repo_grouped_scl_cache.json" + assert repo_cache.exists(), "Repo grouped cache was not created" # Load and verify structure - loaded = load_module_source_links_json(module_cache) + loaded = load_repo_source_links_json(repo_cache) assert isinstance(loaded, list) - assert len(loaded) > 0, "Module cache should contain at least one module" + assert len(loaded) > 0, "Repo cache should contain at least one repo" - # Verify each item is a ModuleSourceLinks + # Verify each item is a RepoSourceLinks for item in loaded: - assert hasattr(item, "module") + assert hasattr(item, "repo") assert hasattr(item, "needs") - assert isinstance(item.module, ModuleInfo) + assert isinstance(item.repo, RepoInfo) finally: app.cleanup() -def test_module_grouping_preserves_metadata( +def test_repo_grouping_preserves_metadata( sphinx_app_setup: Callable[[], SphinxTestApp], sphinx_base_dir: Path, git_repo_setup: Path, create_demo_files: None, ): - """Happy path: Module metadata (name, hash, url) is preserved in cache""" + """Happy path: Repo metadata (name, hash, url) is preserved in cache""" app = sphinx_app_setup() try: os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) app.build() - module_cache = app.outdir / "score_module_grouped_scl_cache.json" - loaded = load_module_source_links_json(module_cache) + repo_cache = app.outdir / "score_repo_grouped_scl_cache.json" + loaded = load_repo_source_links_json(repo_cache) # Verify that each module has proper metadata - for module_links in loaded: - assert module_links.module.name is not None - assert isinstance(module_links.module.name, str) + for repo_links in loaded: + assert repo_links.repo.name is not None + assert isinstance(repo_links.repo.name, str) # Hash and URL might be empty strings for local module - assert module_links.module.hash is not None - assert module_links.module.url is not None + assert repo_links.repo.hash is not None + assert repo_links.repo.url is not None finally: app.cleanup() -def test_module_grouping_multiple_needs_per_module( +def test_repo_grouping_multiple_needs_per_repo( sphinx_app_setup: Callable[[], SphinxTestApp], sphinx_base_dir: Path, git_repo_setup: Path, create_demo_files: None, ): - """Happy path: Multiple needs from same module are grouped together""" + """Happy path: Multiple needs from same repo are grouped together""" app = sphinx_app_setup() try: os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) app.build() - module_cache = app.outdir / "score_module_grouped_scl_cache.json" - loaded = load_module_source_links_json(module_cache) + repo_cache = app.outdir / "score_repo_grouped_scl_cache.json" + loaded = load_repo_source_links_json(repo_cache) - # Find the local_module (should have all 3 requirements) - local_module = None + # Find the local_repo (should have all 3 requirements) + local_repo = None for m in loaded: - if m.module.name == "local_module": - local_module = m + if m.repo.name == "local_repo": + local_repo = m break - assert local_module is not None, "local_module not found in grouped cache" + assert local_repo is not None, "local_repo not found in grouped cache" - # All 3 MOD_REQ should be in this module - need_ids = {need.need for need in local_module.needs} + # All 3 MOD_REQ should be in this repo + need_ids = {need.need for need in local_repo.needs} assert ( "MOD_REQ_1" in need_ids or "MOD_REQ_2" in need_ids @@ -324,14 +332,14 @@ def test_module_grouping_multiple_needs_per_module( app.cleanup() -def test_module_cache_json_format( +def test_repo_cache_json_format( sphinx_app_setup: Callable[[], SphinxTestApp], sphinx_base_dir: Path, git_repo_setup: Path, create_demo_files: None, ): """ - Module cache JSON has correct + Repo cache JSON has correct structure and excludes metadata from links """ app = sphinx_app_setup() @@ -339,30 +347,30 @@ def test_module_cache_json_format( os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) app.build() - module_cache = app.outdir / "score_module_grouped_scl_cache.json" + repo_cache = app.outdir / "score_repo_grouped_scl_cache.json" # Load as raw JSON to check structure - with open(module_cache) as f: + with open(repo_cache) as f: raw_json = json.load(f) assert isinstance(raw_json, list) assert len(raw_json) > 0 - # Check first module structure - first_module = raw_json[0] - assert "module" in first_module - assert "needs" in first_module - assert "name" in first_module["module"] - assert "hash" in first_module["module"] - assert "url" in first_module["module"] + # Check first repo structure + first_repo = raw_json[0] + assert "repo" in first_repo + assert "needs" in first_repo + assert "name" in first_repo["repo"] + assert "hash" in first_repo["repo"] + assert "url" in first_repo["repo"] # Check that needlinks don't have metadata - if first_module["needs"]: - first_need = first_module["needs"][0] + if first_repo["needs"]: + first_need = first_repo["needs"][0] if "links" in first_need and first_need["links"].get("CodeLinks"): codelink = first_need["links"]["CodeLinks"][0] - assert "module_name" not in codelink, ( - "CodeLinks should not contain module_name metadata" + assert "repo_name" not in codelink, ( + "CodeLinks should not contain repo_name metadata" ) assert "hash" not in codelink, ( "CodeLinks should not contain hash metadata" @@ -375,33 +383,33 @@ def test_module_cache_json_format( app.cleanup() -def test_module_cache_rebuilds_when_missing( +def test_repo_cache_rebuilds_when_missing( sphinx_app_setup: Callable[[], SphinxTestApp], sphinx_base_dir: Path, git_repo_setup: Path, create_demo_files: None, ): - """Edge case: Module cache is regenerated if deleted""" + """Edge case: Repo cache is regenerated if deleted""" app = sphinx_app_setup() try: os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) app.build() - module_cache = app.outdir / "score_module_grouped_scl_cache.json" - assert module_cache.exists() + repo_cache = app.outdir / "score_repo_grouped_scl_cache.json" + assert repo_cache.exists() # Delete the cache - module_cache.unlink() - assert not module_cache.exists() + repo_cache.unlink() + assert not repo_cache.exists() # Build again - should regenerate app2 = sphinx_app_setup() app2.build() - assert module_cache.exists(), "Cache should be regenerated on rebuild" + assert repo_cache.exists(), "Cache should be regenerated on rebuild" # Verify it's valid - loaded = load_module_source_links_json(module_cache) + loaded = load_repo_source_links_json(repo_cache) assert len(loaded) > 0 app2.cleanup() @@ -409,50 +417,50 @@ def test_module_cache_rebuilds_when_missing( app.cleanup() -def test_module_grouping_with_golden_file( +def test_repo_grouping_with_golden_file( sphinx_app_setup: Callable[[], SphinxTestApp], sphinx_base_dir: Path, git_repo_setup: Path, create_demo_files: None, ): - """Happy path: Generated module cache matches expected golden file""" + """Happy path: Generated repo cache matches expected golden file""" app = sphinx_app_setup() try: os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) app.build() - module_cache = app.outdir / "score_module_grouped_scl_cache.json" - expected_file = sphinx_base_dir / ".expected_module_grouped.json" + repo_cache = app.outdir / "score_repo_grouped_scl_cache.json" + expected_file = sphinx_base_dir / ".expected_repo_grouped.json" - assert module_cache.exists() + assert repo_cache.exists() assert expected_file.exists(), "Golden file not found" - with open(module_cache) as f1: - actual = json.load(f1, object_hook=ModuleSourceLinks_JSON_Decoder) + with open(repo_cache) as f1: + actual = json.load(f1, object_hook=RepoSourceLinks_JSON_Decoder) with open(expected_file) as f2: - expected = json.load(f2, object_hook=ModuleSourceLinks_JSON_Decoder) + expected = json.load(f2, object_hook=RepoSourceLinks_JSON_Decoder) assert len(actual) == len(expected), ( - f"Module count mismatch. Actual: {len(actual)}, Expected: {len(expected)}" + f"Repo count mismatch. Actual: {len(actual)}, Expected: {len(expected)}" ) - # Compare module by module - actual_by_name = {m.module.name: m for m in actual} - expected_by_name = {m.module.name: m for m in expected} + # Compare repo by repo + actual_by_name = {m.repo.name: m for m in actual} + expected_by_name = {m.repo.name: m for m in expected} assert set(actual_by_name.keys()) == set(expected_by_name.keys()), ( - f"Module names don't match. " + f"Repo names don't match. " f"Actual: {set(actual_by_name.keys())}, " f"Expected: {set(expected_by_name.keys())}" ) - for module_name in actual_by_name: - actual_module = actual_by_name[module_name] - expected_module = expected_by_name[module_name] + for repo_name in actual_by_name: + actual_repo = actual_by_name[repo_name] + expected_repo = expected_by_name[repo_name] - assert actual_module.module.hash == expected_module.module.hash - assert actual_module.module.url == expected_module.module.url - assert len(actual_module.needs) == len(expected_module.needs) + assert actual_repo.repo.hash == expected_repo.repo.hash + assert actual_repo.repo.url == expected_repo.repo.url + assert len(actual_repo.needs) == len(expected_repo.needs) finally: app.cleanup() diff --git a/src/extensions/score_source_code_linker/tests/test_module_source_links.py b/src/extensions/score_source_code_linker/tests/test_repo_source_links.py similarity index 76% rename from src/extensions/score_source_code_linker/tests/test_module_source_links.py rename to src/extensions/score_source_code_linker/tests/test_repo_source_links.py index 293cdc76e..bcb7a7661 100644 --- a/src/extensions/score_source_code_linker/tests/test_module_source_links.py +++ b/src/extensions/score_source_code_linker/tests/test_repo_source_links.py @@ -17,13 +17,13 @@ import pytest -from src.extensions.score_source_code_linker.module_source_links import ( - ModuleInfo, - ModuleSourceLinks, - ModuleSourceLinks_JSON_Decoder, - group_needs_by_module, - load_module_source_links_json, - store_module_source_links_json, +from src.extensions.score_source_code_linker.repo_source_links import ( + RepoInfo, + RepoSourceLinks, + RepoSourceLinks_JSON_Decoder, + group_needs_by_repo, + load_repo_source_links_json, + store_repo_source_links_json, ) from src.extensions.score_source_code_linker.need_source_links import ( NeedSourceLinks, @@ -84,13 +84,13 @@ def SourceCodeLinks_TEST_JSON_Decoder( return d -class ModuleSourceLinks_TEST_JSON_Encoder(json.JSONEncoder): +class repoSourceLinks_TEST_JSON_Encoder(json.JSONEncoder): def default(self, o: object) -> str | dict[str, Any]: if isinstance(o, Path): return str(o) # We do not want to save the metadata inside the codelink or testlink # As we save this already in a structure above it - # (hash, module_name, url) + # (hash, repo_name, url) if isinstance(o, NeedLink | DataForTestLink): d = o.to_dict_without_metadata() tag = d.get("tag", "") @@ -104,9 +104,9 @@ def default(self, o: object) -> str | dict[str, Any]: # dictionaries won't get split up and we will not # run into the 'to_dict_without_metadata' as # everything will be converted to a normal dictionary - if isinstance(o, ModuleSourceLinks): + if isinstance(o, RepoSourceLinks): return { - "module": asdict(o.module), + "repo": asdict(o.repo), "needs": o.needs, # Let the encoder handle the list } if isinstance(o, SourceCodeLinks): @@ -122,17 +122,17 @@ def default(self, o: object) -> str | dict[str, Any]: return super().default(o) -def ModuleSourceLinks_TEST_JSON_Decoder( +def repoSourceLinks_TEST_JSON_Decoder( d: dict[str, Any], -) -> ModuleSourceLinks | dict[str, Any]: - if "module" in d and "needs" in d: - module = d["module"] +) -> RepoSourceLinks | dict[str, Any]: + if "repo" in d and "needs" in d: + repo = d["repo"] needs = d["needs"] - return ModuleSourceLinks( - module=ModuleInfo( - name=module.get("name"), - hash=module.get("hash"), - url=module.get("url"), + return RepoSourceLinks( + repo=RepoInfo( + name=repo.get("name"), + hash=repo.get("hash"), + url=repo.get("url"), ), # We know this can only be list[SourceCodeLinks] and nothing else # Therefore => we ignore the type error here @@ -143,21 +143,21 @@ def ModuleSourceLinks_TEST_JSON_Decoder( def test_json_encoder_removes_metadata_from_needlink(): """Happy path: NeedLink metadata fields are excluded from JSON output""" - encoder = ModuleSourceLinks_TEST_JSON_Encoder() + encoder = repoSourceLinks_TEST_JSON_Encoder() needlink = NeedLink( file=Path("src/test.py"), line=10, tag="#" + " req-Id:", need="REQ_1", full_line="#" + " req-Id: REQ_1", - module_name="test_module", + repo_name="test_repo", url="https://github.com/test/repo", hash="abc123", ) result = encoder.default(needlink) assert isinstance(result, dict) - assert "module_name" not in result + assert "repo_name" not in result assert "url" not in result assert "hash" not in result assert result["need"] == "REQ_1" @@ -166,7 +166,7 @@ def test_json_encoder_removes_metadata_from_needlink(): def test_json_encoder_removes_metadata_from_testlink(): """Happy path: DataForTestLink metadata fields are excluded from JSON output""" - encoder = ModuleSourceLinks_TEST_JSON_Encoder() + encoder = repoSourceLinks_TEST_JSON_Encoder() testlink = DataForTestLink( name="test_something", file=Path("src/test_file.py"), @@ -175,14 +175,14 @@ def test_json_encoder_removes_metadata_from_testlink(): verify_type="fully", result="passed", result_text="", - module_name="test_module", + repo_name="test_repo", url="https://github.com/test/repo", hash="abc123", ) result = encoder.default(testlink) assert isinstance(result, dict) - assert "module_name" not in result + assert "repo_name" not in result assert "url" not in result assert "hash" not in result assert result["name"] == "test_something" @@ -191,7 +191,7 @@ def test_json_encoder_removes_metadata_from_testlink(): def test_json_encoder_converts_path_to_string(): """Happy path: Path objects are converted to strings""" - encoder = ModuleSourceLinks_TEST_JSON_Encoder() + encoder = repoSourceLinks_TEST_JSON_Encoder() result = encoder.default(Path("/test/path/file.py")) assert result == "/test/path/file.py" assert isinstance(result, str) @@ -202,10 +202,10 @@ def test_json_encoder_converts_path_to_string(): # ============================================================================ -def test_json_decoder_reconstructs_module_source_links(): - """Happy path: Valid JSON dict is decoded into ModuleSourceLinks""" +def test_json_decoder_reconstructs_repo_source_links(): + """Happy path: Valid JSON dict is decoded into repoSourceLinks""" json_data: dict[str, Any] = { - "module": {"name": "test_module", "hash": "hash1", "url": "url1"}, + "repo": {"name": "test_repo", "hash": "hash1", "url": "url1"}, "needs": [ { "need": "REQ_1", @@ -213,19 +213,19 @@ def test_json_decoder_reconstructs_module_source_links(): } ], } - result = ModuleSourceLinks_JSON_Decoder(json_data) + result = RepoSourceLinks_JSON_Decoder(json_data) - assert isinstance(result, ModuleSourceLinks) - assert result.module.name == "test_module" - assert result.module.hash == "hash1" - assert result.module.url == "url1" + assert isinstance(result, RepoSourceLinks) + assert result.repo.name == "test_repo" + assert result.repo.hash == "hash1" + assert result.repo.url == "url1" assert len(result.needs) == 1 -def test_json_decoder_returns_unchanged_for_non_module_dict(): - """Edge case: Non-ModuleSourceLinks dicts are returned unchanged""" +def test_json_decoder_returns_unchanged_for_non_repo_dict(): + """Edge case: Non-repoSourceLinks dicts are returned unchanged""" json_data = {"some": "data", "other": "values"} - result = ModuleSourceLinks_JSON_Decoder(json_data) + result = RepoSourceLinks_JSON_Decoder(json_data) assert result == json_data @@ -236,14 +236,14 @@ def test_json_decoder_returns_unchanged_for_non_module_dict(): def test_store_and_load_roundtrip(tmp_path: Path): """Happy path: Store and load preserves data correctly""" - module = ModuleInfo(name="test_module", hash="hash1", url="url1") + repo = RepoInfo(name="test_repo", hash="hash1", url="url1") needlink = NeedLink( file=Path("src/test.py"), line=10, tag="#" + " req-Id:", need="REQ_1", full_line="#" + " req-Id: REQ_1", - module_name="test_module", + repo_name="test_repo", url="url1", hash="hash1", ) @@ -251,24 +251,24 @@ def test_store_and_load_roundtrip(tmp_path: Path): need="REQ_1", links=NeedSourceLinks(CodeLinks=[needlink], TestLinks=[]), ) - module_links = ModuleSourceLinks(module=module, needs=[scl]) + repo_links = RepoSourceLinks(repo=repo, needs=[scl]) test_file = tmp_path / "test.json" - store_module_source_links_json(test_file, [module_links]) - loaded = load_module_source_links_json(test_file) + store_repo_source_links_json(test_file, [repo_links]) + loaded = load_repo_source_links_json(test_file) assert len(loaded) == 1 - assert loaded[0].module.name == "test_module" + assert loaded[0].repo.name == "test_repo" assert loaded[0].needs[0].need == "REQ_1" def test_store_creates_parent_directories(tmp_path: Path): """Edge case: Parent directories are created if they don't exist""" nested_path = tmp_path / "nested" / "deeply" / "test.json" - module = ModuleInfo(name="test", hash="h", url="u") - module_links = ModuleSourceLinks(module=module, needs=[]) + repo = RepoInfo(name="test", hash="h", url="u") + repo_links = RepoSourceLinks(repo=repo, needs=[]) - store_module_source_links_json(nested_path, [module_links]) + store_repo_source_links_json(nested_path, [repo_links]) assert nested_path.exists() assert nested_path.parent.exists() @@ -277,19 +277,19 @@ def test_store_creates_parent_directories(tmp_path: Path): def test_load_empty_list(tmp_path: Path): """Edge case: Loading empty list returns empty list""" test_file = tmp_path / "empty.json" - store_module_source_links_json(test_file, []) + store_repo_source_links_json(test_file, []) - loaded = load_module_source_links_json(test_file) + loaded = load_repo_source_links_json(test_file) assert loaded == [] def test_load_validates_is_list(tmp_path: Path): """Bad path: Loading non-list JSON fails validation""" test_file = tmp_path / "not_list.json" - test_file.write_text('{"module": {}, "needs": []}') + test_file.write_text('{"repo": {}, "needs": []}') with pytest.raises(AssertionError, match="should be a list"): - load_module_source_links_json(test_file) + load_repo_source_links_json(test_file) def test_load_validates_items_are_correct_type(tmp_path: Path): @@ -297,24 +297,24 @@ def test_load_validates_items_are_correct_type(tmp_path: Path): test_file = tmp_path / "invalid_items.json" test_file.write_text('[{"invalid": "structure"}]') - with pytest.raises(AssertionError, match="should be ModuleSourceLink objects"): - load_module_source_links_json(test_file) + with pytest.raises(AssertionError, match="should be RepoSourceLink objects"): + load_repo_source_links_json(test_file) # ============================================================================ -# group_needs_by_module Tests +# group_needs_by_repo Tests # ============================================================================ -def test_group_needs_single_module_with_codelinks(): - """Happy path: Multiple needs from same module are grouped together""" +def test_group_needs_single_repo_with_codelinks(): + """Happy path: Multiple needs from same repo are grouped together""" needlink1 = NeedLink( file=Path("src/file1.py"), line=10, tag="#" + " req-Id:", need="REQ_1", full_line="#" + " req-Id: REQ_1", - module_name="shared_module", + repo_name="shared_repo", url="https://github.com/test/repo", hash="hash1", ) @@ -324,7 +324,7 @@ def test_group_needs_single_module_with_codelinks(): tag="#" + " req-Id:", need="REQ_2", full_line="#" + " req-Id: REQ_2", - module_name="shared_module", + repo_name="shared_repo", url="https://github.com/test/repo", hash="hash1", ) @@ -336,25 +336,25 @@ def test_group_needs_single_module_with_codelinks(): need="REQ_2", links=NeedSourceLinks(CodeLinks=[needlink2], TestLinks=[]) ) - result = group_needs_by_module([scl1, scl2]) + result = group_needs_by_repo([scl1, scl2]) assert len(result) == 1 - assert result[0].module.name == "shared_module" - assert result[0].module.hash == "hash1" + assert result[0].repo.name == "shared_repo" + assert result[0].repo.hash == "hash1" assert len(result[0].needs) == 2 assert len(result[0].needs[0].links.CodeLinks) == 1 assert len(result[0].needs[1].links.CodeLinks) == 1 -def test_group_needs_multiple_modules(): - """Happy path: Needs from different modules create separate groups""" +def test_group_needs_multiple_repos(): + """Happy path: Needs from different repos create separate groups""" needlink_a = NeedLink( file=Path("src/a.py"), line=10, tag="#" + " req-Id:", need="REQ_1", full_line="#" + " req-Id: REQ_1", - module_name="module_a", + repo_name="repo_a", url="https://github.com/a/repo", hash="hash_a", ) @@ -364,7 +364,7 @@ def test_group_needs_multiple_modules(): tag="#" + " req-Id:", need="REQ_2", full_line="#" + " req-Id: REQ_2", - module_name="module_b", + repo_name="repo_b", url="https://github.com/b/repo", hash="hash_b", ) @@ -376,15 +376,15 @@ def test_group_needs_multiple_modules(): need="REQ_2", links=NeedSourceLinks(CodeLinks=[needlink_b], TestLinks=[]) ) - result = group_needs_by_module([scl1, scl2]) + result = group_needs_by_repo([scl1, scl2]) assert len(result) == 2 - assert result[0].module.name == "module_a" - assert result[0].module.hash == "hash_a" - assert result[0].module.url == "https://github.com/a/repo" - assert result[1].module.name == "module_b" - assert result[1].module.hash == "hash_b" - assert result[1].module.url == "https://github.com/b/repo" + assert result[0].repo.name == "repo_a" + assert result[0].repo.hash == "hash_a" + assert result[0].repo.url == "https://github.com/a/repo" + assert result[1].repo.name == "repo_b" + assert result[1].repo.hash == "hash_b" + assert result[1].repo.url == "https://github.com/b/repo" def test_group_needs_with_testlinks_only(): @@ -397,7 +397,7 @@ def test_group_needs_with_testlinks_only(): verify_type="fully", result="passed", result_text="", - module_name="test_module", + repo_name="test_repo", url="https://github.com/test/repo", hash="hash1", ) @@ -409,7 +409,7 @@ def test_group_needs_with_testlinks_only(): verify_type="partially", result="passed", result_text="", - module_name="test_module", + repo_name="test_repo", url="https://github.com/test/repo", hash="hash1", ) @@ -423,10 +423,10 @@ def test_group_needs_with_testlinks_only(): links=NeedSourceLinks(CodeLinks=[], TestLinks=[testlink2]), ) - result = group_needs_by_module([scl, scl2]) + result = group_needs_by_repo([scl, scl2]) assert len(result) == 1 - assert result[0].module.name == "test_module" + assert result[0].repo.name == "test_repo" assert len(result[0].needs) == 2 needs = [x.need for x in result[0].needs] assert "REQ_1" in needs @@ -435,8 +435,8 @@ def test_group_needs_with_testlinks_only(): assert len(result[0].needs[1].links.TestLinks) == 1 -def test_group_needs_with_testlinks_different_modules(): - """Testing Testlinks grouping for different modules""" +def test_group_needs_with_testlinks_different_repos(): + """Testing Testlinks grouping for different repos""" testlink = DataForTestLink( name="test_feature", file=Path("tests/test.py"), @@ -445,7 +445,7 @@ def test_group_needs_with_testlinks_different_modules(): verify_type="fully", result="passed", result_text="", - module_name="module_a", + repo_name="repo_a", url="https://github.com/test_a/repo_a", hash="hash_a", ) @@ -457,7 +457,7 @@ def test_group_needs_with_testlinks_different_modules(): verify_type="partially", result="passed", result_text="", - module_name="module_b", + repo_name="repo_b", url="https://github.com/test_b/repo_b", hash="hash_b", ) @@ -471,20 +471,20 @@ def test_group_needs_with_testlinks_different_modules(): links=NeedSourceLinks(CodeLinks=[], TestLinks=[testlink2]), ) - result = group_needs_by_module([scl, scl2]) + result = group_needs_by_repo([scl, scl2]) assert len(result) == 2 - assert result[0].module.name == "module_a" - assert result[0].module.hash == "hash_a" - assert result[0].module.url == "https://github.com/test_a/repo_a" - assert result[1].module.name == "module_b" - assert result[1].module.hash == "hash_b" - assert result[1].module.url == "https://github.com/test_b/repo_b" + assert result[0].repo.name == "repo_a" + assert result[0].repo.hash == "hash_a" + assert result[0].repo.url == "https://github.com/test_a/repo_a" + assert result[1].repo.name == "repo_b" + assert result[1].repo.hash == "hash_b" + assert result[1].repo.url == "https://github.com/test_b/repo_b" def test_group_needs_empty_list(): """Edge case: Empty list returns empty result""" - result = group_needs_by_module([]) + result = group_needs_by_repo([]) assert result == [] @@ -504,7 +504,7 @@ def test_group_needs_skips_needs_without_links(): tag="#" + " req-Id:", need="REQ_1", full_line="#" + " req-Id: REQ_1", - module_name="module_a", + repo_name="repo_a", url="url1", hash="hash1", ) @@ -517,7 +517,7 @@ def test_group_needs_skips_needs_without_links(): links=NeedSourceLinks(CodeLinks=[], TestLinks=[]), ) - result = group_needs_by_module([scl_with_links, scl_without_links]) + result = group_needs_by_repo([scl_with_links, scl_without_links]) assert len(result) == 1 assert result[0].needs[0].need == "REQ_1" @@ -532,7 +532,7 @@ def test_group_needs_mixed_codelinks_and_testlinks(): tag="#" + " req-Id:", need="REQ_1", full_line="#" + " req-Id: REQ_1", - module_name="module_a", + repo_name="repo_a", url="https://github.com/test/repo", hash="hash1", ) @@ -544,7 +544,7 @@ def test_group_needs_mixed_codelinks_and_testlinks(): verify_type="fully", result="passed", result_text="", - module_name="module_a", + repo_name="repo_a", url="https://github.com/test/repo", hash="hash1", ) @@ -554,7 +554,7 @@ def test_group_needs_mixed_codelinks_and_testlinks(): links=NeedSourceLinks(CodeLinks=[needlink], TestLinks=[testlink]), ) - result = group_needs_by_module([scl]) + result = group_needs_by_repo([scl]) assert len(result) == 1 assert len(result[0].needs[0].links.CodeLinks) == 1 diff --git a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py index b6e1ded0b..a92d96311 100644 --- a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py +++ b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py @@ -26,7 +26,7 @@ from sphinx_needs.data import SphinxNeedsData from src.extensions.score_source_code_linker.helpers import get_github_link -from src.extensions.score_source_code_linker.module_source_links import ModuleInfo +from src.extensions.score_source_code_linker.repo_source_links import RepoInfo from src.extensions.score_source_code_linker.needlinks import NeedLink from src.extensions.score_source_code_linker.testlink import ( DataForTestLink, @@ -313,7 +313,7 @@ def example_source_link_text_all_ok(sphinx_base_dir: Path) -> dict[str, list[Nee tag="#" + " req-Id:", need="TREQ_ID_1", full_line="#" + " req-Id: TREQ_ID_1", - module_name="local_module", + repo_name="local_repo", url="", hash="", ), @@ -323,7 +323,7 @@ def example_source_link_text_all_ok(sphinx_base_dir: Path) -> dict[str, list[Nee tag="#" + " req-Id:", need="TREQ_ID_1", full_line="#" + " req-Id: TREQ_ID_1", - module_name="local_module", + repo_name="local_repo", url="", hash="", ), @@ -335,7 +335,7 @@ def example_source_link_text_all_ok(sphinx_base_dir: Path) -> dict[str, list[Nee tag="#" + " req-Id:", need="TREQ_ID_2", full_line="#" + " req-Id: TREQ_ID_2", - module_name="local_module", + repo_name="local_repo", url="", hash="", ) @@ -356,7 +356,7 @@ def example_test_link_text_all_ok(sphinx_base_dir: Path): verify_type="fully", result="passed", result_text="", - module_name="local_module", + repo_name="local_repo", url="", hash="", ), @@ -370,7 +370,7 @@ def example_test_link_text_all_ok(sphinx_base_dir: Path): verify_type="partially", result="passed", result_text="", - module_name="local_module", + repo_name="local_repo", url="", hash="", ), @@ -382,7 +382,7 @@ def example_test_link_text_all_ok(sphinx_base_dir: Path): verify_type="partially", result="passed", result_text="", - module_name="local_module", + repo_name="local_repo", url="", hash="", ), @@ -396,7 +396,7 @@ def example_test_link_text_all_ok(sphinx_base_dir: Path): verify_type="partially", result="passed", result_text="", - module_name="local_module", + repo_name="local_repo", url="", hash="", ), @@ -408,7 +408,7 @@ def example_test_link_text_all_ok(sphinx_base_dir: Path): verify_type="partially", result="passed", result_text="", - module_name="local_module", + repo_name="local_repo", url="", hash="", ), @@ -434,21 +434,23 @@ def example_source_link_text_non_existent(sphinx_base_dir: Path): def make_source_link(needlinks: list[NeedLink]): - metadata= ModuleInfo( - name="local_module", - url="", - hash="", + metadata = RepoInfo( + name="local_repo", + url="", + hash="", + ) + return ", ".join( + f"{get_github_link(metadata, n)}<>{n.file}:{n.line}" for n in needlinks ) - return ", ".join(f"{get_github_link(metadata,n)}<>{n.file}:{n.line}" for n in needlinks) def make_test_link(testlinks: list[DataForTestLink]): - metadata= ModuleInfo( - name="local_module", - url="", - hash="", + metadata = RepoInfo( + name="local_repo", + url="", + hash="", ) - return ", ".join(f"{get_github_link(metadata,n)}<>{n.name}" for n in testlinks) + return ", ".join(f"{get_github_link(metadata, n)}<>{n.name}" for n in testlinks) def compare_json_files( diff --git a/src/extensions/score_source_code_linker/tests/test_testlink.py b/src/extensions/score_source_code_linker/tests/test_testlink.py index 7e49b8983..9d3e8ecab 100644 --- a/src/extensions/score_source_code_linker/tests/test_testlink.py +++ b/src/extensions/score_source_code_linker/tests/test_testlink.py @@ -53,7 +53,7 @@ def test_testlink_serialization_roundtrip(): result="passed", result_text="All good", # ADAPTED: Added new fields - module_name="test_module", + repo_name="test_repo", hash="abc12345", url="https://github.com/org/repo", ) @@ -117,7 +117,7 @@ def test_testcaseneed_to_dict_multiple_links(): PartiallyVerifies="REQ-1, REQ-2", FullyVerifies="REQ-3", # ADAPTED: Added new fields which are now required for valid TestLinks - module_name="test_module", + repo_name="test_repo", hash="hash123", url="http://github.com", ) @@ -134,7 +134,7 @@ def test_testcaseneed_to_dict_multiple_links(): assert link.name == "TC_01" assert link.result == "failed" # ADAPTED: Verify new fields are propagated - assert link.module_name == "test_module" + assert link.repo_name == "test_repo" assert link.hash == "hash123" assert link.url == "http://github.com" @@ -158,7 +158,7 @@ def test_store_and_load_testlinks_roundtrip(tmp_path: Path): result="passed", result_text="Looks good", # ADAPTED: Added new fields - module_name="mod1", + repo_name="mod1", hash="h1", url="u1", ), @@ -171,7 +171,7 @@ def test_store_and_load_testlinks_roundtrip(tmp_path: Path): result="failed", result_text="Needs work", # ADAPTED: Added new fields - module_name="mod2", + repo_name="mod2", hash="h2", url="u2", ), @@ -197,13 +197,13 @@ def test_datafortestlink_to_dict_full(): verify_type="fully", result="passed", result_text="All good", - module_name="test_module", + repo_name="test_repo", hash="abc123", url="https://github.com/test/repo", ) result = link.to_dict_full() - assert result["module_name"] == "test_module" + assert result["repo_name"] == "test_repo" assert result["hash"] == "abc123" assert result["url"] == "https://github.com/test/repo" assert result["name"] == "test_full" @@ -239,7 +239,7 @@ def test_dataoftestcase_check_verifies_fields_missing_both(): result_text="", PartiallyVerifies=None, FullyVerifies=None, - module_name="mod", + repo_name="mod", hash="h", url="u", ) @@ -258,7 +258,7 @@ def test_dataoftestcase_is_valid_fails_on_none_field(): DerivationTechnique="manual", result_text="", PartiallyVerifies="REQ_1", - module_name="mod", + repo_name="mod", hash="h", url="u", ) @@ -294,7 +294,7 @@ def test_dataoftestcase_decoder_valid_dict(): "result_text": "Good", "PartiallyVerifies": "REQ_1", "FullyVerifies": "REQ_2", - "module_name": "mod", + "repo_name": "mod", "hash": "h", "url": "u", } @@ -325,7 +325,7 @@ def test_store_and_load_data_of_test_case_roundtrip(tmp_path: Path): result_text="OK", PartiallyVerifies="REQ_A", FullyVerifies=None, - module_name="mod_a", + repo_name="mod_a", hash="hash_a", url="url_a", ), @@ -339,7 +339,7 @@ def test_store_and_load_data_of_test_case_roundtrip(tmp_path: Path): result_text="Failed", PartiallyVerifies=None, FullyVerifies="REQ_B", - module_name="mod_b", + repo_name="mod_b", hash="hash_b", url="url_b", ), diff --git a/src/extensions/score_source_code_linker/tests/test_xml_parser.py b/src/extensions/score_source_code_linker/tests/test_xml_parser.py index 3c328fedf..7f0438b31 100644 --- a/src/extensions/score_source_code_linker/tests/test_xml_parser.py +++ b/src/extensions/score_source_code_linker/tests/test_xml_parser.py @@ -269,7 +269,7 @@ def test_parse_properties(): derivation_technique="requirements-analysis", ) # ADAPTED: Added patching for metadata functions -@patch("src.extensions.score_source_code_linker.xml_parser.parse_module_name_from_path") +@patch("src.extensions.score_source_code_linker.xml_parser.parse_repo_name_from_path") def test_read_test_xml_file( mock_parse_module: Any, tmp_xml_dirs: Callable[..., tuple[Path, Path, Path, Path, Path]], @@ -278,7 +278,7 @@ def test_read_test_xml_file( # ADAPTED: Mock return value to ensure metadata is populated. # 'local_module' triggers the path where hash/url are empty strings, # which is valid but avoids the need to mock parse_info_from_known_good. - mock_parse_module.return_value = "local_module" + mock_parse_module.return_value = "local_repo" _: Path dir1: Path @@ -291,7 +291,7 @@ def test_read_test_xml_file( assert isinstance(tcneed, DataOfTestCase) assert tcneed.result == "failed" # ADAPTED: Verify metadata fields were populated - assert tcneed.module_name == "local_module" + assert tcneed.repo_name == "local_repo" assert tcneed.hash == "" assert tcneed.url == "" diff --git a/src/extensions/score_source_code_linker/xml_parser.py b/src/extensions/score_source_code_linker/xml_parser.py index d44f0854c..2aa791056 100644 --- a/src/extensions/score_source_code_linker/xml_parser.py +++ b/src/extensions/score_source_code_linker/xml_parser.py @@ -36,10 +36,11 @@ from src.extensions.score_source_code_linker.helpers import ( get_github_link, parse_info_from_known_good, - parse_module_name_from_path, + parse_repo_name_from_path, ) -from src.extensions.score_source_code_linker.module_source_links import ModuleInfo +from src.extensions.score_source_code_linker.repo_source_links import RepoInfo from src.extensions.score_source_code_linker.needlinks import ( + DefaultMetaData, MetaData, ) from src.extensions.score_source_code_linker.testlink import ( @@ -82,15 +83,15 @@ def get_metadata_from_test_path(raw_filepath: Path) -> MetaData: Will parse out the metadata from the testpath. If test is local then the metadata will be: - "module_name": "local_module", + "repo_name": "local_repo", "hash": "", "url": "", - Else it will parse the module_name e.g. `score_docs_as_code` + Else it will parse the repo_name e.g. `score_docs_as_code` match this in the known_good_json and grab the accompanying hash, url as well and return metadata like so for example: - "module_name": "score_docs_as_code", + "repo_name": "score_docs_as_code", "hash": "c1207676afe6cafd25c35d420e73279a799515d8", "url": "https://github.com/eclipse-score/docs-as-code" @@ -110,15 +111,12 @@ def get_metadata_from_test_path(raw_filepath: Path) -> MetaData: known_good_json = os.environ.get("KNOWN_GOOD_JSON") clean_filepath = clean_test_file_name(raw_filepath) # print(f"This is the cleaned filepath: {clean_filepath}") - module_name = parse_module_name_from_path(clean_filepath) - md: MetaData = { - "module_name": module_name, - "hash": "", - "url": "", - } - if module_name != "local_module" and known_good_json: + repo_name = parse_repo_name_from_path(clean_filepath) + md = DefaultMetaData() + md["repo_name"] = repo_name + if repo_name != "local_repo" and known_good_json: md["hash"], md["url"] = parse_info_from_known_good( - Path(known_good_json), module_name + Path(known_good_json), repo_name ) return md @@ -346,11 +344,11 @@ def construct_and_add_need(app: Sphinx, tn: DataOfTestCase): # and either 'Fully' or 'PartiallyVerifies' should not be None here assert tn.file is not None assert tn.name is not None - assert tn.module_name is not None + assert tn.repo_name is not None assert tn.hash is not None assert tn.url is not None # Have to build metadata here for the gh link func - metadata = ModuleInfo(name=tn.module_name, hash=tn.hash, url=tn.url) + metadata = RepoInfo(name=tn.repo_name, hash=tn.hash, url=tn.url) # IDK if this is ideal or not with contextlib.suppress(BaseException): _ = add_external_need( From ae5969df84b953a6e52a7c26a7780daa5f516e69 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Fri, 20 Mar 2026 10:55:35 +0100 Subject: [PATCH 27/34] Formating & Linting --- src/extensions/score_source_code_linker/__init__.py | 10 +++++----- src/extensions/score_source_code_linker/helpers.py | 3 +-- .../score_source_code_linker/repo_source_links.py | 2 -- .../score_source_code_linker/tests/test_codelink.py | 4 +--- .../score_source_code_linker/tests/test_helpers.py | 2 +- .../tests/test_repo_source_links.py | 10 +++++----- .../tests/test_source_code_link_integration.py | 2 +- src/extensions/score_source_code_linker/xml_parser.py | 2 +- 8 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/extensions/score_source_code_linker/__init__.py b/src/extensions/score_source_code_linker/__init__.py index 59d336ceb..24a2a1bee 100644 --- a/src/extensions/score_source_code_linker/__init__.py +++ b/src/extensions/score_source_code_linker/__init__.py @@ -35,11 +35,6 @@ generate_source_code_links_json, ) from src.extensions.score_source_code_linker.helpers import get_github_link -from src.extensions.score_source_code_linker.repo_source_links import ( - group_needs_by_repo, - load_repo_source_links_json, - store_repo_source_links_json, -) from src.extensions.score_source_code_linker.need_source_links import ( group_by_need, load_source_code_links_combined_json, @@ -49,6 +44,11 @@ load_source_code_links_json, load_source_code_links_with_metadata_json, ) +from src.extensions.score_source_code_linker.repo_source_links import ( + group_needs_by_repo, + load_repo_source_links_json, + store_repo_source_links_json, +) from src.extensions.score_source_code_linker.testlink import ( load_data_of_test_case_json, load_test_xml_parsed_json, diff --git a/src/extensions/score_source_code_linker/helpers.py b/src/extensions/score_source_code_linker/helpers.py index b41b7cc56..5311717eb 100644 --- a/src/extensions/score_source_code_linker/helpers.py +++ b/src/extensions/score_source_code_linker/helpers.py @@ -13,10 +13,9 @@ import json from pathlib import Path -from src.extensions.score_source_code_linker.repo_source_links import RepoInfo - # Import types that depend on score_source_code_linker from src.extensions.score_source_code_linker.needlinks import DefaultNeedLink, NeedLink +from src.extensions.score_source_code_linker.repo_source_links import RepoInfo from src.extensions.score_source_code_linker.testlink import ( DataForTestLink, DataOfTestCase, diff --git a/src/extensions/score_source_code_linker/repo_source_links.py b/src/extensions/score_source_code_linker/repo_source_links.py index 841c20780..4fbd2adaa 100644 --- a/src/extensions/score_source_code_linker/repo_source_links.py +++ b/src/extensions/score_source_code_linker/repo_source_links.py @@ -152,5 +152,3 @@ def group_needs_by_repo(links: list[SourceCodeLinks]) -> list[RepoSourceLinks]: RepoSourceLinks(repo=group.repo, needs=group.needs) for group in repo_groups.values() ] - - diff --git a/src/extensions/score_source_code_linker/tests/test_codelink.py b/src/extensions/score_source_code_linker/tests/test_codelink.py index 25387e7ec..b6a277335 100644 --- a/src/extensions/score_source_code_linker/tests/test_codelink.py +++ b/src/extensions/score_source_code_linker/tests/test_codelink.py @@ -23,8 +23,6 @@ from dataclasses import asdict from pathlib import Path from typing import Any -from collections import Counter -from collections.abc import Callable import pytest @@ -45,7 +43,6 @@ from src.extensions.score_source_code_linker.helpers import ( get_github_link, ) -from src.extensions.score_source_code_linker.repo_source_links import RepoInfo from src.extensions.score_source_code_linker.needlinks import ( MetaData, NeedLink, @@ -57,6 +54,7 @@ store_source_code_links_json, store_source_code_links_with_metadata_json, ) +from src.extensions.score_source_code_linker.repo_source_links import RepoInfo from src.helper_lib import ( get_current_git_hash, ) diff --git a/src/extensions/score_source_code_linker/tests/test_helpers.py b/src/extensions/score_source_code_linker/tests/test_helpers.py index d55e4517b..86fca65ad 100644 --- a/src/extensions/score_source_code_linker/tests/test_helpers.py +++ b/src/extensions/score_source_code_linker/tests/test_helpers.py @@ -25,8 +25,8 @@ parse_info_from_known_good, parse_repo_name_from_path, ) -from src.extensions.score_source_code_linker.repo_source_links import RepoInfo from src.extensions.score_source_code_linker.needlinks import DefaultNeedLink +from src.extensions.score_source_code_linker.repo_source_links import RepoInfo from src.helper_lib import get_current_git_hash # ╭──────────────────────────────────────────────────────────╮ diff --git a/src/extensions/score_source_code_linker/tests/test_repo_source_links.py b/src/extensions/score_source_code_linker/tests/test_repo_source_links.py index bcb7a7661..33a3d0384 100644 --- a/src/extensions/score_source_code_linker/tests/test_repo_source_links.py +++ b/src/extensions/score_source_code_linker/tests/test_repo_source_links.py @@ -17,6 +17,11 @@ import pytest +from src.extensions.score_source_code_linker.need_source_links import ( + NeedSourceLinks, + SourceCodeLinks, +) +from src.extensions.score_source_code_linker.needlinks import NeedLink from src.extensions.score_source_code_linker.repo_source_links import ( RepoInfo, RepoSourceLinks, @@ -25,11 +30,6 @@ load_repo_source_links_json, store_repo_source_links_json, ) -from src.extensions.score_source_code_linker.need_source_links import ( - NeedSourceLinks, - SourceCodeLinks, -) -from src.extensions.score_source_code_linker.needlinks import NeedLink from src.extensions.score_source_code_linker.testlink import DataForTestLink """ diff --git a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py index a92d96311..9d604a43a 100644 --- a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py +++ b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py @@ -26,8 +26,8 @@ from sphinx_needs.data import SphinxNeedsData from src.extensions.score_source_code_linker.helpers import get_github_link -from src.extensions.score_source_code_linker.repo_source_links import RepoInfo from src.extensions.score_source_code_linker.needlinks import NeedLink +from src.extensions.score_source_code_linker.repo_source_links import RepoInfo from src.extensions.score_source_code_linker.testlink import ( DataForTestLink, DataForTestLink_JSON_Decoder, diff --git a/src/extensions/score_source_code_linker/xml_parser.py b/src/extensions/score_source_code_linker/xml_parser.py index 2aa791056..f6472a9f8 100644 --- a/src/extensions/score_source_code_linker/xml_parser.py +++ b/src/extensions/score_source_code_linker/xml_parser.py @@ -38,11 +38,11 @@ parse_info_from_known_good, parse_repo_name_from_path, ) -from src.extensions.score_source_code_linker.repo_source_links import RepoInfo from src.extensions.score_source_code_linker.needlinks import ( DefaultMetaData, MetaData, ) +from src.extensions.score_source_code_linker.repo_source_links import RepoInfo from src.extensions.score_source_code_linker.testlink import ( DataOfTestCase, store_data_of_test_case_json, From 78b0095b00934871944d33947ecddcb27435aceb Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Mon, 23 Mar 2026 10:21:50 +0100 Subject: [PATCH 28/34] Rename helpers file --- src/extensions/score_draw_uml_funcs/__init__.py | 2 +- .../score_draw_uml_funcs/{helpers.py => draw_helpers.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/extensions/score_draw_uml_funcs/{helpers.py => draw_helpers.py} (100%) diff --git a/src/extensions/score_draw_uml_funcs/__init__.py b/src/extensions/score_draw_uml_funcs/__init__.py index d4a95f39d..3388463b7 100644 --- a/src/extensions/score_draw_uml_funcs/__init__.py +++ b/src/extensions/score_draw_uml_funcs/__init__.py @@ -32,7 +32,7 @@ from pathlib import Path from typing import Any, cast -from score_draw_uml_funcs.helpers import ( +from score_draw_uml_funcs.draw_helpers import ( gen_header, gen_interface_element, gen_link_text, diff --git a/src/extensions/score_draw_uml_funcs/helpers.py b/src/extensions/score_draw_uml_funcs/draw_helpers.py similarity index 100% rename from src/extensions/score_draw_uml_funcs/helpers.py rename to src/extensions/score_draw_uml_funcs/draw_helpers.py From 9c35436dc7ce03760fb641eb399b0c5118e03620 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Mon, 23 Mar 2026 15:57:16 +0100 Subject: [PATCH 29/34] Fix: Bugfixing action Incremental was passed into every build target regardless if it was correct or not --- docs.bzl | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs.bzl b/docs.bzl index 9eca87261..a7a730e83 100644 --- a/docs.bzl +++ b/docs.bzl @@ -189,13 +189,11 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good = docs_env = { "SOURCE_DIRECTORY": source_dir, "DATA": str(data), - "ACTION": "incremental", "SCORE_SOURCELINKS": "$(location :sourcelinks_json)", } docs_sources_env = { "SOURCE_DIRECTORY": source_dir, "DATA": str(data_with_docs_sources), - "ACTION": "incremental", "SCORE_SOURCELINKS": "$(location :merged_sourcelinks)", } if known_good: @@ -204,15 +202,18 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good = docs_data.append(known_good) combo_data.append(known_good) + docs_env["ACTION"] = "incremental" + py_binary( name = "docs", tags = ["cli_help=Build documentation:\nbazel run //:docs"], srcs = ["@score_docs_as_code//src:incremental.py"], data = docs_data, deps = deps, - env = docs_env, + env = docs_env ) + docs_sources_env["ACTION"] = "incremental" py_binary( name = "docs_combo", tags = ["cli_help=Build full documentation with all dependencies:\nbazel run //:docs_combo"], @@ -228,6 +229,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good = deprecation = "Target '//:docs_combo_experimental' is deprecated. Use '//:docs_combo' instead.", ) + docs_env["ACTION"] = "linkcheck" py_binary( name = "docs_link_check", tags = ["cli_help=Verify Links inside Documentation:\nbazel run //:link_check\n (Note: this could take a long time)"], @@ -237,24 +239,27 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good = env = docs_env ) + docs_env["ACTION"] = "check" py_binary( name = "docs_check", tags = ["cli_help=Verify documentation:\nbazel run //:docs_check"], srcs = ["@score_docs_as_code//src:incremental.py"], data = docs_data, deps = deps, - env = docs_env, + env = docs_env ) + docs_env["ACTION"] = "live_preview" py_binary( name = "live_preview", tags = ["cli_help=Live preview documentation in the browser:\nbazel run //:live_preview"], srcs = ["@score_docs_as_code//src:incremental.py"], data = docs_data, deps = deps, - env = docs_env, + env = docs_env ) + docs_sources_env["ACTION"] = "live_preview" py_binary( name = "live_preview_combo_experimental", tags = ["cli_help=Live preview full documentation with all dependencies in the browser:\nbazel run //:live_preview_combo_experimental"], From fdfeb2be850092fbf4113948c7274e1779b62486 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Mon, 23 Mar 2026 17:21:22 +0100 Subject: [PATCH 30/34] Remove emoji headers --- src/extensions/docs/source_code_linker.md | 321 +++++++++++++--------- 1 file changed, 196 insertions(+), 125 deletions(-) diff --git a/src/extensions/docs/source_code_linker.md b/src/extensions/docs/source_code_linker.md index e97c4b6f9..d283f1c7f 100644 --- a/src/extensions/docs/source_code_linker.md +++ b/src/extensions/docs/source_code_linker.md @@ -8,64 +8,118 @@ This extension integrates with **Bazel** and **sphinx-needs** to automatically g ## Overview -The extension is split into two main components: +The feature is split into **two layers**: -- **CodeLink** – Parses source files for template strings and links them to needs. -- **TestLink** – Parses test.xml outputs inside `bazel-testlogs` to link test cases to requirements. +1. **Bazel pre-processing (before Sphinx runs)** + Generates and aggregates JSON caches containing the raw `source_code_link` findings (and, if available, repository metadata like `repo_name/hash/url`). -Each component stores intermediate data in JSON caches under `_build/` to optimize performance and speed up incremental builds. +2. **Sphinx extension (during the Sphinx build)** + Reads the aggregated JSON, validates and adapts it, and finally injects the links into Sphinx needs in the final layout (**RepoSourceLink**). + +This separation makes combo builds faster and more deterministic, because the expensive repository scanning/parsing happens **outside** the Sphinx process. --- ## How It Works -### ✅ CodeLink: Source Code Integration +### Bazel Pre-processing: Source Code Integration (CodeLinks) + +The Bazel parts are responsible for producing the **intermediate caches** that the source_code_linker extension later will consumes. -CodeLink scans repository files (excluding `_`, `.`, and binary formats) for requirement tags such as: +#### Step 1: Per-repository cache generation +All files provided via the `scan_code` attribute to the docs macro will be scanned for the requirement tags. +A *per repository JSON cache* will be generated and saved. +This script `scripts_bazel/generate_sourcelinks_cli.py` finds all codelinks per file, and gathers them into +one JSON cache per repository. +It also adds metadata to each needlink that is needed in further steps. + +Example of requirement tags: ```python # Choose one or the other, both mentioned here to avoid detection # req-Id/req-traceability: ``` -These tags are extracted and matched to Sphinx needs via the `source_code_link` attribute. If a need ID does not exist, a build warning will be raised. - -#### Data Flow - -1. **File Scanning** (`generate_source_code_links_json.py`) - - Filters out files starting with `_`, `.`, or ending in `.pyc`, `.so`, `.exe`, `.bin`. - - Searches for template tags: `# req-Id:` and `# req-traceability:`. - - Extracts: - - File path - - Line number - - Tag and full line - - Associated need ID - - Saves data as JSON via `needlinks.py`. - -2. **Link Creation** - - Git info (file hash) is used to build a GitHub URL to the line in the source file. - - Links are injected into needs via the `source_code_link` attribute during the Sphinx build process. +- Extracts for each match: + - File path + - Line number + - Tag + full line + - Associated need ID + - Repository Metadata: + - repo_name (parsed from filepath) (local_repo if non combo build) + - hash (at this stage always blank) + - url (at this stage always blank) -#### Example JSON Cache (CodeLinks) +##### Example JSON cache (per repository) -``` +```json [ { "file": "src/extensions/score_metamodel/metamodel.yaml", "line": 17, - "tag": "#--req-Id:", # added `--` to avoid detection + "tag": "#--req-Id:", "need": "tool_req__docs_dd_link_source_code_link", - "full_line": "#--req-Id: tool_req__docs_dd_link_source_code_link" # added `--` to avoid detection + "full_line": "#--req-Id: tool_req__docs_dd_link_source_code_link", + "repo_name": "local_repo", + "hash": "", + "url": "" } ] ``` +> Note: `--` is shown in the examples to avoid accidental detection by the parser. + +--- + +#### Step 2: Cache merge step (multi-repo aggregation) + +In a second Bazel step `scripts_bazel/merge_sourcelinks.py`, **all per-repo caches** are merged into a **single combined JSON**. + +- Input: N JSON caches (one per repo) +- Output: 1 merged JSON containing all found source_code_links + +This step also fills in url & hash if there is a known_good_json provided (e.g. in a combo build) + + +#### Repo metadata rules +Here are some basic rules regarding the MetaData information + +In a combo build, a known_good_json **must** be provided. +If known_good_json is found then the hash & url will be filled for each need by the script. + +Combo build: + - `repo_name`: repository name (parsed from filepath in step 1) + - `hash`: revision/commit hash (as provided by the known_good_json) + - `url`: repository remote URL (as provided by the known_good_json) + +Local build: + - `repo_name`: 'local_repo' + - `hash`: will be empty at this point, and later filled via parsing the git commands + - `url`: will be empty at this point, and later filled via parsing the git commands + --- -### ✅ TestLink: Test Result Integration +### Source Code Linker Extension + +1a. **Reads the merged JSON** produced by Bazel +1b. **Reads test.xml files and generates testlinks JSON** +2. **Validates and adapts** both JSON cache files +3. **Merges and converts** them into the final structure: **RepoSourceLink** -TestLink scans test result XMLs from Bazel (bazel-testlogs) or in the folder 'tests-report' and converts each test case with metadata into Sphinx external needs, allowing links from tests to requirements. +#### Codelinks + +Since these are all found in [Bazel step 1](#step-1-per-repository-cache-generation) this essentially does not run +anymore inside of the source_code_linker extension. + +#### Testlinks + +TestLink scans test result XMLs from Bazel (bazel-testlogs) or from the folder 'tests-report' and converts each test case with metadata into Sphinx external needs, allowing links from tests to requirements. This depends on the `attribute_plugin` in our tooling repository, find it [here](https://github.com/eclipse-score/tooling/tree/main/python_basics/score_pytest) + +:::attention +If TestLinks should be generated in a combo build please ensure that you have the known_good_json added to the docs macro. +::: + #### Test Tagging Options ```python @@ -90,7 +144,7 @@ def test_feature(): #### Data Flow 1. **XML Parsing** (`xml_parser.py`) - - Scans `bazel-testlogs/` for `test.xml` files. + - Scans `bazel-testlogs/` or `tests-report` for `test.xml` files. - Parses test cases and extracts: - Name & Classname - File path @@ -98,6 +152,7 @@ def test_feature(): - Result (e.g. passed, failed, skipped) - Result text (if failed/skipped will check if message was attached to it) - Verifications (`PartiallyVerifies`, `FullyVerifies`) + - Also parses metadata according to the [metadata rules](#repo-metadata-rules) - Cases without metadata are logged out as info (not errors). - Test cases with metadata are converted into: @@ -125,17 +180,23 @@ The DataFromTestCase depicts the information gathered about one testcase. "result_text": "", "PartiallyVerifies": "tool_req__docs_common_attr_title, tool_req__docs_common_attr_description", "FullyVerifies": null + "repo_name": "local_module", + "hash": "", + "url": "", } ] ``` --- -## 🔗 Combined Links -During the Sphinx build process, both CodeLink and TestLink data are combined and applied to needs. +### Early-Combined Links + +During the Sphinx build process, both CodeLink and TestLink data are combined and grouped by needs. +This will allow us to have an easier time building the final JSON which will eliminate the metadata from each link, +and group all needs according to the appropriate repository. -This is handled in `__init__.py` using the `NeedSourceLinks` and `SourceCodeLinks` dataclasses from `need_source_links.py`. +This all is handled in `need_source_links.py`. ### Combined JSON Example @@ -150,7 +211,11 @@ This is handled in `__init__.py` using the `NeedSourceLinks` and `SourceCodeLink "line": 33, "tag": "#--req-Id:",# added `--` to avoid detection "need": "tool_req__docs_common_attr_title", - "full_line": "#--req-Id: tool_req__docs_common_attr_title" # added `--` to avoid detection + "full_line": "#--req-Id: tool_req__docs_common_attr_title", # added `--` to avoid detection + "repo_name": "local_module", + "hash": "", + "url": "", + } ], "TestLinks": [ @@ -161,7 +226,10 @@ This is handled in `__init__.py` using the `NeedSourceLinks` and `SourceCodeLink "need": "tool_req__docs_common_attr_title", "verify_type": "partially", "result": "passed", - "result_text": "" + "result_text": "", + "repo_name": "local_module", + "hash": "", + "url": "", } ] } @@ -169,36 +237,117 @@ This is handled in `__init__.py` using the `NeedSourceLinks` and `SourceCodeLink ] ``` +#### Final layout: RepoSourceLink + +Instead of repeating `repo_name/hash/url` for every single link entry, the final output groups links **by repository**: + +- Repository metadata appears **once per repository** +- All links belonging to that repository are stored beneath it + +This somewhat looks like this: +```json +[ + { + "repo": { + "name": "local_repo", + "hash": "", + "url": "" + }, + "needs": [ + { + "need": "tool_req__docs_common_attr_id_scheme", + "links": { + "CodeLinks": [ + { + "file": "src/extensions/score_metamodel/checks/attributes_format.py", + "line": 30, + "tag": "# req-Id:", + "need": "tool_req__docs_common_attr_id_scheme", + "full_line": "# req-Id: tool_req__docs_common_attr_id_scheme" + } + ], + "TestLinks": ... + } + }, + ], + { ... } +``` +Due to not saving the `repo_name, url and hash` in each link but grouping them we can eleminate a lot of unnecessary +length to the JSON document here. + +This is the structure the extension uses to attach links to needs (via the `source_code_link` attribute), while keeping the resulting data compact and normalized. + +:::hint +Currently if the repo is the local one, SCL is still using the hash and url it gets from executing the git functions parsing. +This is a known inefficiency and will be improved upon later +::: + +--- + +## Result: Traceability Links in Needs + +During the Sphinx build process, the extension applies the computed links to needs: + +- Each need’s `source_code_link` and `testlink` attribute is filled from the (repo-grouped) RepoSourceLink data where applicable. +- If a referenced need ID does not exist, a build warning will be raised. --- -## ⚠️ Known Limitations +## Known Limitations -### CodeLink +### General +- In combo builds, known_good_json is required. +- inefficiencies in creating the links and saving the JSON caches +- Not compatible with **Esbonio/Live_preview** -- ❌ Not compatible with **Esbonio/Live_preview** -- 🔗 GitHub links may 404 if the commit isn’t pushed -- 🧪 Tags must match exactly (e.g. # req-Id) -- 👀 `source_code_link` isn’t visible until the full Sphinx build is completed +### Codelinks +- GitHub links may 404 if the commit isn’t pushed +- Tags must match exactly (e.g. # req-Id) +- `source_code_link` isn’t visible until the full Sphinx build is completed ### TestLink -- ❌ Not compatible with **Esbonio/Live_preview** -- 🔗 GitHub links may 404 if the commit isn’t pushed -- 🧪 XML structure must be followed exactly (e.g. `properties & attributes`) -- 🗂 Relies on test to be executed first +- GitHub links may 404 if the commit isn’t pushed +- XML structure must be followed exactly (e.g. `properties & attributes`) +- Relies on test to be executed first +--- + +## High-level Data Flow Summary +1. **Bazel Script #1**: scan + parse → write **per-repo cache** (includes repo metadata if known) +2. **Bazel Script #2**: merge caches → write **single merged JSON** +3. **Sphinx Extension**: read merged JSON → adapt to **RepoSourceLink** → inject source_code_link and testlink into needs --- -## 🏗️ Internal Module Overview +## Clearing Cache Manually + +To clear the build cache, run: +```bash +rm -rf _build/ +``` + +## Internal Overview + +The bazel part: +``` +scripts_bazel/ +├── BUILD # Declare libraries and filegroups needed for bazel +├── generate_sourcelinks_cli.py # Bazel step 1 => Parses sourcefiles for tags +├── merge_sourcelinks.py +└── tests +│ └── ... +``` +The Sphinx extension ``` score_source_code_linker/ ├── __init__.py # Main Sphinx extension; combines CodeLinks + TestLinks -├── generate_source_code_links_json.py # Parses source files for tags +├── generate_source_code_links_json.py # Most functionality moved to 'scripts_bazel/generate_sourcelinks_cli' ├── need_source_links.py # Data model for combined links +├── repo_source_links.py # Data model for Repo combined links (Final output JSON) +├── helpers.py # Misc. functions used throughout SCL ├── needlinks.py # CodeLink dataclass & JSON encoder/decoder ├── testlink.py # DataForTestLink definition & logic ├── xml_parser.py # Parses XML files into test case data @@ -222,81 +371,3 @@ To see working examples for CodeLinks & TestLinks, take a look at the Docs-As-Co [Example CodeLink](https://eclipse-score.github.io/docs-as-code/main/requirements/requirements.html#tool_req__docs_common_attr_status) [Example TestLink](https://eclipse-score.github.io/docs-as-code/main/requirements/requirements.html#tool_req__docs_dd_link_source_code_link) - -## Flow-Overview -```{mermaid} -flowchart TD - %% Entry Point - A[source_code_linker] --> B{Check for Grouped JSON Cache} - - %% If cache exists - B --> |✅| C[Load Grouped JSON Cache] - B --> |🔴| N9[Proceed Without Cache] - - %% --- NeedLink Path --- - N9 --> D1[needslink.py
NeedLink] - D1 --> E1{Check for CodeLink JSON Cache} - - E1 --> |✅| F1[Load CodeLink JSON Cache] - F1 --> Z[Grouped JSON Cache] - - E1 --> |🔴| G1[Parse all files in repository] - G1 --> H1[Build & Save
CodeLink JSON Cache] - H1 --> Z - - %% --- TestLink Path --- - N9 --> D2[testlink.py
DFTL] - D2 --> E2{Check for DFTL JSON Cache} - - E2 --> |✅| F2[Load DFTL JSON Cache] - F2 --> J2[Load DOTC JSON Cache] - J2 --> K2[Add as External Needs] - - E2 --> |🔴| G2[Parse test.xml Files] - G2 --> H2[Convert TestCases
to DOTC] - H2 --> I2[Build & Save
DOTC JSON Cache] - I2 --> K2 - - H2 --> M2[Convert to DFTL] - M2 --> N2[Build & Save
DFTL JSON Cache] - N2 --> Z - - %% Final step - Z --> FINAL[Add links to needs] - - %% Legend - subgraph Legend["Legend"] - direction TB - L1[NeedLink Operations] - L2[TestLink Operations] - L4[DTFL = DataForTestLink] - L3[TestCaseNeed Operations] - L5[DOTC = DataOfTestCase] - L1 ~~~ L2 - L2 ~~~ L4 - L4 ~~~ L3 - L3 ~~~ L5 - end - - %% Node Styling - classDef needlink fill:#3b82f6,color:#ffffff,stroke:#1d4ed8,stroke-width:2px - classDef testlink fill:#8b5cf6,color:#ffffff,stroke:#6d28d9,stroke-width:2px - classDef dotc fill:#f59e0b,color:#ffffff,stroke:#b45309,stroke-width:2px - classDef grouped fill:#10b981,color:#ffffff,stroke:#047857,stroke-width:2px - classDef final fill:#f43f5e,color:#ffffff,stroke:#be123c,stroke-width:2px - - %% Class assignments - class D1,E1,F1,G1,H1 needlink - class D2,E2,F2,G2,M2,N2 testlink - class J2,H2,I2,K2 dotc - class Z grouped - class FINAL final - class L1 needlink - class L2,L4 testlink - class L3,L5 dotc - - %% Edge/Arrow Styling - linkStyle default stroke:#22d3ee,stroke-width:2px,color:#22d3ee - %% Ensure links in the Legend do not show up - linkStyle 23,24,25,26 opacity:0 -``` From 8c7aee6b3b9b7f18ae5c5ce3625fbcbc6c41b550 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Thu, 26 Mar 2026 09:25:29 +0100 Subject: [PATCH 31/34] Bugfix, id's should not be uppercase --- src/extensions/score_source_code_linker/xml_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extensions/score_source_code_linker/xml_parser.py b/src/extensions/score_source_code_linker/xml_parser.py index f6472a9f8..ad00c9360 100644 --- a/src/extensions/score_source_code_linker/xml_parser.py +++ b/src/extensions/score_source_code_linker/xml_parser.py @@ -356,7 +356,7 @@ def construct_and_add_need(app: Sphinx, tn: DataOfTestCase): need_type="testcase", title=tn.name, tags="TEST", - id=f"testcase__{tn.name}_{short_hash(tn.file + tn.name).upper()}", + id=f"testcase__{tn.name}_{short_hash(tn.file + tn.name)}", name=tn.name, external_url=get_github_link(metadata, tn), fully_verifies=tn.FullyVerifies if tn.FullyVerifies is not None else "", From 9473852d9ea838d6bc92b69b71951c3c1577291a Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Thu, 26 Mar 2026 09:42:31 +0100 Subject: [PATCH 32/34] Fix myst errors --- src/extensions/docs/source_code_linker.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/extensions/docs/source_code_linker.md b/src/extensions/docs/source_code_linker.md index d283f1c7f..f6f827669 100644 --- a/src/extensions/docs/source_code_linker.md +++ b/src/extensions/docs/source_code_linker.md @@ -26,6 +26,7 @@ This separation makes combo builds faster and more deterministic, because the ex The Bazel parts are responsible for producing the **intermediate caches** that the source_code_linker extension later will consumes. +(step-1-per-repository-cache-generation)= #### Step 1: Per-repository cache generation All files provided via the `scan_code` attribute to the docs macro will be scanned for the requirement tags. @@ -52,7 +53,7 @@ Example of requirement tags: ##### Example JSON cache (per repository) -```json +```{code-block} json [ { "file": "src/extensions/score_metamodel/metamodel.yaml", @@ -80,7 +81,7 @@ In a second Bazel step `scripts_bazel/merge_sourcelinks.py`, **all per-repo cach This step also fills in url & hash if there is a known_good_json provided (e.g. in a combo build) - +(repo-metadata-rules)= #### Repo metadata rules Here are some basic rules regarding the MetaData information @@ -245,7 +246,7 @@ Instead of repeating `repo_name/hash/url` for every single link entry, the final - All links belonging to that repository are stored beneath it This somewhat looks like this: -```json +```{code-block} json [ { "repo": { @@ -266,11 +267,10 @@ This somewhat looks like this: "full_line": "# req-Id: tool_req__docs_common_attr_id_scheme" } ], - "TestLinks": ... + "TestLinks": [] } }, ], - { ... } ``` Due to not saving the `repo_name, url and hash` in each link but grouping them we can eleminate a lot of unnecessary length to the JSON document here. From feb98632754466426547a7d24d2a46b3f13bc1d5 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Thu, 26 Mar 2026 10:16:19 +0100 Subject: [PATCH 33/34] Add config value --- src/extensions/score_source_code_linker/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/extensions/score_source_code_linker/__init__.py b/src/extensions/score_source_code_linker/__init__.py index 24a2a1bee..29628b5ca 100644 --- a/src/extensions/score_source_code_linker/__init__.py +++ b/src/extensions/score_source_code_linker/__init__.py @@ -301,6 +301,7 @@ def setup_once(app: Sphinx): def setup(app: Sphinx) -> dict[str, str | bool]: # Esbonio will execute setup() on every iteration. # setup_once will only be called once. + app.add_config_value("KNOWN_GOOD_JSON", default="",rebuild="env",types=str) setup_once(app) return { From 2d58de23747cdd103e47fd34bd30c87fbdd62577 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Thu, 26 Mar 2026 12:11:15 +0100 Subject: [PATCH 34/34] Fix several small things found by copilot Fixing assertion Fixing wrong split in path Fixing pytest global arguments --- pyproject.toml | 1 - .../score_source_code_linker/tests/test_helpers.py | 1 + .../tests/test_repo_source_link_integration.py | 11 ++++++----- .../score_source_code_linker/tests/test_xml_parser.py | 6 +++--- src/extensions/score_source_code_linker/xml_parser.py | 4 ++-- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8dd24d16b..6aa48c6f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,6 @@ extend-exclude = [ ".venv*/**", ] [tool.pytest.ini_options] -addopts = ["-v", "-s"] log_cli = true log_cli_level = "Debug" log_cli_format = "[%(asctime)s.%(msecs)03d] [%(levelname)-3s] [%(name)s] %(message)s" diff --git a/src/extensions/score_source_code_linker/tests/test_helpers.py b/src/extensions/score_source_code_linker/tests/test_helpers.py index 86fca65ad..ab5207bb3 100644 --- a/src/extensions/score_source_code_linker/tests/test_helpers.py +++ b/src/extensions/score_source_code_linker/tests/test_helpers.py @@ -323,6 +323,7 @@ def test_get_github_link_with_real_repo(git_repo: Path) -> None: assert hash_from_link == actual_hash + def test_complete_workflow(known_good_json: Path): """Test complete workflow from path to GitHub link.""" diff --git a/src/extensions/score_source_code_linker/tests/test_repo_source_link_integration.py b/src/extensions/score_source_code_linker/tests/test_repo_source_link_integration.py index 270fef076..79360e902 100644 --- a/src/extensions/score_source_code_linker/tests/test_repo_source_link_integration.py +++ b/src/extensions/score_source_code_linker/tests/test_repo_source_link_integration.py @@ -10,6 +10,11 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* + +# ╓ ╖ +# ║ Some portions generated by CoPilot ║ +# ╙ ╜ + import contextlib import json import os @@ -322,11 +327,7 @@ def test_repo_grouping_multiple_needs_per_repo( # All 3 MOD_REQ should be in this repo need_ids = {need.need for need in local_repo.needs} - assert ( - "MOD_REQ_1" in need_ids - or "MOD_REQ_2" in need_ids - or "MOD_REQ_3" in need_ids - ) + assert {"MOD_REQ_1", "MOD_REQ_2", "MOD_REQ_3"}.issubset(need_ids) finally: app.cleanup() diff --git a/src/extensions/score_source_code_linker/tests/test_xml_parser.py b/src/extensions/score_source_code_linker/tests/test_xml_parser.py index 7f0438b31..3a2e110c7 100644 --- a/src/extensions/score_source_code_linker/tests/test_xml_parser.py +++ b/src/extensions/score_source_code_linker/tests/test_xml_parser.py @@ -359,7 +359,7 @@ def test_clean_test_file_name_combo_path(): def test_clean_test_file_name_tests_report_path(): raw_path = Path("/some/path/tests-report/test-report/unit/test_module.xml") result = xml_parser.clean_test_file_name(raw_path) - assert result == Path("unit/test_module.xml") + assert result == Path("test-report/unit/test_module.xml") def test_clean_test_file_name_nested_bazel_testlogs(): @@ -371,7 +371,7 @@ def test_clean_test_file_name_nested_bazel_testlogs(): def test_clean_test_file_name_invalid_path_raises_error(): raw_path = Path("/invalid/path/without/markers/test.xml") with pytest.raises( - ValueError, match="Filepath does not have 'bazel-testlogs' nor 'test-report'" + ValueError, match="Filepath does not have 'bazel-testlogs' nor 'tests-report'" ): xml_parser.clean_test_file_name(raw_path) @@ -379,6 +379,6 @@ def test_clean_test_file_name_invalid_path_raises_error(): def test_clean_test_file_name_empty_path_raises_error(): raw_path = Path("") with pytest.raises( - ValueError, match="Filepath does not have 'bazel-testlogs' nor 'test-report'" + ValueError, match="Filepath does not have 'bazel-testlogs' nor 'tests-report'" ): xml_parser.clean_test_file_name(raw_path) diff --git a/src/extensions/score_source_code_linker/xml_parser.py b/src/extensions/score_source_code_linker/xml_parser.py index ad00c9360..cfa41f7f3 100644 --- a/src/extensions/score_source_code_linker/xml_parser.py +++ b/src/extensions/score_source_code_linker/xml_parser.py @@ -71,10 +71,10 @@ def clean_test_file_name(raw_filepath: Path) -> Path: if "bazel-testlogs" in str(raw_filepath): return Path(str(raw_filepath).split("bazel-testlogs/")[-1]) if "tests-report" in str(raw_filepath): - return Path(str(raw_filepath).split("test-report/")[-1]) + return Path(str(raw_filepath).split("tests-report/")[-1]) raise ValueError( "Filepath does not have 'bazel-testlogs' nor " - f"'test-report'. Filepath: {raw_filepath}" + + f"'tests-report'. Filepath: {raw_filepath}" )