diff --git a/docs.bzl b/docs.bzl index cd41a4c83..a7a730e83 100644 --- a/docs.bzl +++ b/docs.bzl @@ -69,7 +69,7 @@ def _rewrite_needs_json_to_sourcelinks(labels): out.append(s) return out -def _merge_sourcelinks(name, sourcelinks): +def _merge_sourcelinks(name, sourcelinks, known_good = None): """Merge multiple sourcelinks JSON files into a single file. Args: @@ -77,15 +77,22 @@ def _merge_sourcelinks(name, sourcelinks): sourcelinks: List of sourcelinks JSON file targets """ + extra_srcs = [] + known_good_arg = "" + if known_good != None: + extra_srcs = [known_good] + known_good_arg = "--known_good $(location %s)" % known_good + native.genrule( name = name, - srcs = sourcelinks, + srcs = sourcelinks + extra_srcs, outs = [name + ".json"], cmd = """ $(location @score_docs_as_code//scripts_bazel:merge_sourcelinks) \ --output $@ \ + {known_good_arg} \ $(SRCS) - """, + """.format(known_good_arg = known_good_arg), tools = ["@score_docs_as_code//scripts_bazel:merge_sourcelinks"], ) @@ -120,7 +127,7 @@ def _missing_requirements(deps): fail(msg) fail("This case should be unreachable?!") -def docs(source_dir = "docs", data = [], deps = [], scan_code = []): +def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good = None): """Creates all targets related to documentation. By using this function, you'll get any and all updates for documentation targets in one place. @@ -175,34 +182,45 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []): data_with_docs_sources = _rewrite_needs_json_to_docs_sources(data) additional_combo_sourcelinks = _rewrite_needs_json_to_sourcelinks(data) - _merge_sourcelinks(name = "merged_sourcelinks", sourcelinks = [":sourcelinks_json"] + additional_combo_sourcelinks) + _merge_sourcelinks(name = "merged_sourcelinks", sourcelinks = [":sourcelinks_json"] + additional_combo_sourcelinks, known_good = known_good) + docs_data = data + [":sourcelinks_json"] + combo_data = data_with_docs_sources + [":merged_sourcelinks"] + + docs_env = { + "SOURCE_DIRECTORY": source_dir, + "DATA": str(data), + "SCORE_SOURCELINKS": "$(location :sourcelinks_json)", + } + docs_sources_env = { + "SOURCE_DIRECTORY": source_dir, + "DATA": str(data_with_docs_sources), + "SCORE_SOURCELINKS": "$(location :merged_sourcelinks)", + } + if known_good: + docs_env["KNOWN_GOOD_JSON"] = "$(location "+ known_good + ")" + docs_sources_env["KNOWN_GOOD_JSON"] = "$(location "+ known_good + ")" + docs_data.append(known_good) + combo_data.append(known_good) + + docs_env["ACTION"] = "incremental" py_binary( name = "docs", tags = ["cli_help=Build documentation:\nbazel run //:docs"], srcs = ["@score_docs_as_code//src:incremental.py"], - data = data + [":sourcelinks_json"], + data = docs_data, deps = deps, - env = { - "SOURCE_DIRECTORY": source_dir, - "DATA": str(data), - "ACTION": "incremental", - "SCORE_SOURCELINKS": "$(location :sourcelinks_json)", - }, + env = docs_env ) + docs_sources_env["ACTION"] = "incremental" py_binary( name = "docs_combo", tags = ["cli_help=Build full documentation with all dependencies:\nbazel run //:docs_combo"], srcs = ["@score_docs_as_code//src:incremental.py"], - data = data_with_docs_sources + [":merged_sourcelinks"], + data = combo_data, deps = deps, - env = { - "SOURCE_DIRECTORY": source_dir, - "DATA": str(data_with_docs_sources), - "ACTION": "incremental", - "SCORE_SOURCELINKS": "$(location :merged_sourcelinks)", - }, + env = docs_sources_env ) native.alias( @@ -211,59 +229,44 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = []): deprecation = "Target '//:docs_combo_experimental' is deprecated. Use '//:docs_combo' instead.", ) + docs_env["ACTION"] = "linkcheck" py_binary( name = "docs_link_check", tags = ["cli_help=Verify Links inside Documentation:\nbazel run //:link_check\n (Note: this could take a long time)"], srcs = ["@score_docs_as_code//src:incremental.py"], - data = data, + data = docs_data, deps = deps, - env = { - "SOURCE_DIRECTORY": source_dir, - "DATA": str(data), - "ACTION": "linkcheck", - }, + env = docs_env ) + docs_env["ACTION"] = "check" py_binary( name = "docs_check", tags = ["cli_help=Verify documentation:\nbazel run //:docs_check"], srcs = ["@score_docs_as_code//src:incremental.py"], - data = data + [":sourcelinks_json"], + data = docs_data, deps = deps, - env = { - "SOURCE_DIRECTORY": source_dir, - "DATA": str(data), - "ACTION": "check", - "SCORE_SOURCELINKS": "$(location :sourcelinks_json)", - }, + env = docs_env ) + docs_env["ACTION"] = "live_preview" py_binary( name = "live_preview", tags = ["cli_help=Live preview documentation in the browser:\nbazel run //:live_preview"], srcs = ["@score_docs_as_code//src:incremental.py"], - data = data + [":sourcelinks_json"], + data = docs_data, deps = deps, - env = { - "SOURCE_DIRECTORY": source_dir, - "DATA": str(data), - "ACTION": "live_preview", - "SCORE_SOURCELINKS": "$(location :sourcelinks_json)", - }, + env = docs_env ) + docs_sources_env["ACTION"] = "live_preview" py_binary( name = "live_preview_combo_experimental", tags = ["cli_help=Live preview full documentation with all dependencies in the browser:\nbazel run //:live_preview_combo_experimental"], srcs = ["@score_docs_as_code//src:incremental.py"], - data = data_with_docs_sources + [":merged_sourcelinks"], + data = combo_data, deps = deps, - env = { - "SOURCE_DIRECTORY": source_dir, - "DATA": str(data_with_docs_sources), - "ACTION": "live_preview", - "SCORE_SOURCELINKS": "$(location :merged_sourcelinks)", - }, + env = docs_sources_env ) score_virtualenv( diff --git a/scripts_bazel/BUILD b/scripts_bazel/BUILD index 81c9212f1..befe51730 100644 --- a/scripts_bazel/BUILD +++ b/scripts_bazel/BUILD @@ -33,6 +33,7 @@ py_binary( py_binary( name = "merge_sourcelinks", srcs = ["merge_sourcelinks.py"], + deps= [ "//src/extensions/score_source_code_linker"], main = "merge_sourcelinks.py", visibility = ["//visibility:public"], ) diff --git a/scripts_bazel/generate_sourcelinks_cli.py b/scripts_bazel/generate_sourcelinks_cli.py index 4291b97c5..e5b134a45 100644 --- a/scripts_bazel/generate_sourcelinks_cli.py +++ b/scripts_bazel/generate_sourcelinks_cli.py @@ -25,25 +25,43 @@ from src.extensions.score_source_code_linker.generate_source_code_links_json import ( _extract_references_from_file, # pyright: ignore[reportPrivateUsage] TODO: move it out of the extension and into this script ) +from src.extensions.score_source_code_linker.helpers import parse_repo_name_from_path from src.extensions.score_source_code_linker.needlinks import ( - store_source_code_links_json, + DefaultMetaData, + store_source_code_links_with_metadata_json, ) logging.basicConfig(level=logging.INFO, format="%(message)s") logger = logging.getLogger(__name__) +def clean_external_prefix(path: Path) -> Path: + """ + As it can be in combo builds that the path is: + `external/score_docs_as_code+/....` or similar + We have to remove this prefix from the file + before we pass it to the extract function. Otherwise we have + this prefix inside the `file` attribute leading to wrong links + """ + if not str(path).startswith("external/"): + return path + # This allows for files / folders etc. to have `external` in their name too. + path_raw = str(path).removeprefix("external/") + filepath_split = str(path_raw).split("/", maxsplit=1) + return Path("".join(filepath_split[1:])) + + def main(): parser = argparse.ArgumentParser( description="Generate source code links JSON from source files" ) - parser.add_argument( + _ = parser.add_argument( "--output", required=True, type=Path, help="Output JSON file path", ) - parser.add_argument( + _ = parser.add_argument( "files", nargs="*", type=Path, @@ -53,15 +71,23 @@ def main(): args = parser.parse_args() all_need_references = [] + + metadata = DefaultMetaData() + metadata_set = False for file_path in args.files: + if "known_good.json" not in str(file_path) and not metadata_set: + metadata["repo_name"] = parse_repo_name_from_path(file_path) + metadata_set = True abs_file_path = file_path.resolve() assert abs_file_path.exists(), abs_file_path + clean_path = clean_external_prefix(file_path) references = _extract_references_from_file( - abs_file_path.parent, Path(abs_file_path.name) + abs_file_path.parent, Path(abs_file_path.name), clean_path ) all_need_references.extend(references) - - store_source_code_links_json(args.output, all_need_references) + store_source_code_links_with_metadata_json( + file=args.output, metadata=metadata, needlist=all_need_references + ) logger.info( f"Found {len(all_need_references)} need references in {len(args.files)} files" ) diff --git a/scripts_bazel/merge_sourcelinks.py b/scripts_bazel/merge_sourcelinks.py index f194e19ca..88499d11a 100644 --- a/scripts_bazel/merge_sourcelinks.py +++ b/scripts_bazel/merge_sourcelinks.py @@ -21,6 +21,8 @@ import sys from pathlib import Path +from src.extensions.score_source_code_linker.helpers import parse_info_from_known_good + logging.basicConfig(level=logging.INFO, format="%(message)s") logger = logging.getLogger(__name__) @@ -29,13 +31,18 @@ def main(): parser = argparse.ArgumentParser( description="Merge multiple sourcelinks JSON files into one" ) - parser.add_argument( + _ = parser.add_argument( "--output", required=True, type=Path, help="Output merged JSON file path", ) - parser.add_argument( + _ = parser.add_argument( + "--known_good", + required=True, + help="Path to a required 'known good' JSON file (provided by Bazel).", + ) + _ = parser.add_argument( "files", nargs="*", type=Path, @@ -43,14 +50,38 @@ def main(): ) args = parser.parse_args() + all_files = [x for x in args.files if "known_good.json" not in str(x)] merged = [] - for json_file in args.files: + for json_file in all_files: with open(json_file) as f: data = json.load(f) - assert isinstance(data, list), repr(data) - merged.extend(data) + # If the file is empty e.g. '[]' there is nothing to parse, we continue + if not data: + continue + metadata = data[0] + if not isinstance(metadata, dict) or "repo_name" not in metadata: + logger.warning( + f"Unexpected schema in sourcelinks file '{json_file}': " + "expected first element to be a metadata dict " + "with a 'repo_name' key. " + ) + # As we can't deal with bad JSON structure we just skip it + continue + if metadata["repo_name"] and metadata["repo_name"] != "local_repo": + hash, repo = parse_info_from_known_good( + known_good_json=args.known_good, repo_name=metadata["repo_name"] + ) + metadata["hash"] = hash + metadata["url"] = repo + # In the case that 'metadata[repo_name]' is 'local_module' + # hash & url are already existing and empty inside of 'metadata' + # Therefore all 3 keys will be written to needlinks in each branch + for d in data[1:]: + d.update(metadata) + assert isinstance(data, list), repr(data) + merged.extend(data[1:]) with open(args.output, "w") as f: json.dump(merged, f, indent=2, ensure_ascii=False) diff --git a/src/extensions/docs/source_code_linker.md b/src/extensions/docs/source_code_linker.md index e97c4b6f9..f6f827669 100644 --- a/src/extensions/docs/source_code_linker.md +++ b/src/extensions/docs/source_code_linker.md @@ -8,64 +8,119 @@ This extension integrates with **Bazel** and **sphinx-needs** to automatically g ## Overview -The extension is split into two main components: +The feature is split into **two layers**: -- **CodeLink** โ€“ Parses source files for template strings and links them to needs. -- **TestLink** โ€“ Parses test.xml outputs inside `bazel-testlogs` to link test cases to requirements. +1. **Bazel pre-processing (before Sphinx runs)** + Generates and aggregates JSON caches containing the raw `source_code_link` findings (and, if available, repository metadata like `repo_name/hash/url`). -Each component stores intermediate data in JSON caches under `_build/` to optimize performance and speed up incremental builds. +2. **Sphinx extension (during the Sphinx build)** + Reads the aggregated JSON, validates and adapts it, and finally injects the links into Sphinx needs in the final layout (**RepoSourceLink**). + +This separation makes combo builds faster and more deterministic, because the expensive repository scanning/parsing happens **outside** the Sphinx process. --- ## How It Works -### โœ… CodeLink: Source Code Integration +### Bazel Pre-processing: Source Code Integration (CodeLinks) + +The Bazel parts are responsible for producing the **intermediate caches** that the source_code_linker extension later will consumes. -CodeLink scans repository files (excluding `_`, `.`, and binary formats) for requirement tags such as: +(step-1-per-repository-cache-generation)= +#### Step 1: Per-repository cache generation +All files provided via the `scan_code` attribute to the docs macro will be scanned for the requirement tags. +A *per repository JSON cache* will be generated and saved. +This script `scripts_bazel/generate_sourcelinks_cli.py` finds all codelinks per file, and gathers them into +one JSON cache per repository. +It also adds metadata to each needlink that is needed in further steps. + +Example of requirement tags: ```python # Choose one or the other, both mentioned here to avoid detection # req-Id/req-traceability: ``` -These tags are extracted and matched to Sphinx needs via the `source_code_link` attribute. If a need ID does not exist, a build warning will be raised. - -#### Data Flow - -1. **File Scanning** (`generate_source_code_links_json.py`) - - Filters out files starting with `_`, `.`, or ending in `.pyc`, `.so`, `.exe`, `.bin`. - - Searches for template tags: `# req-Id:` and `# req-traceability:`. - - Extracts: - - File path - - Line number - - Tag and full line - - Associated need ID - - Saves data as JSON via `needlinks.py`. - -2. **Link Creation** - - Git info (file hash) is used to build a GitHub URL to the line in the source file. - - Links are injected into needs via the `source_code_link` attribute during the Sphinx build process. +- Extracts for each match: + - File path + - Line number + - Tag + full line + - Associated need ID + - Repository Metadata: + - repo_name (parsed from filepath) (local_repo if non combo build) + - hash (at this stage always blank) + - url (at this stage always blank) -#### Example JSON Cache (CodeLinks) +##### Example JSON cache (per repository) -``` +```{code-block} json [ { "file": "src/extensions/score_metamodel/metamodel.yaml", "line": 17, - "tag": "#--req-Id:", # added `--` to avoid detection + "tag": "#--req-Id:", "need": "tool_req__docs_dd_link_source_code_link", - "full_line": "#--req-Id: tool_req__docs_dd_link_source_code_link" # added `--` to avoid detection + "full_line": "#--req-Id: tool_req__docs_dd_link_source_code_link", + "repo_name": "local_repo", + "hash": "", + "url": "" } ] ``` +> Note: `--` is shown in the examples to avoid accidental detection by the parser. + --- -### โœ… TestLink: Test Result Integration +#### Step 2: Cache merge step (multi-repo aggregation) + +In a second Bazel step `scripts_bazel/merge_sourcelinks.py`, **all per-repo caches** are merged into a **single combined JSON**. + +- Input: N JSON caches (one per repo) +- Output: 1 merged JSON containing all found source_code_links + +This step also fills in url & hash if there is a known_good_json provided (e.g. in a combo build) + +(repo-metadata-rules)= +#### Repo metadata rules +Here are some basic rules regarding the MetaData information + +In a combo build, a known_good_json **must** be provided. +If known_good_json is found then the hash & url will be filled for each need by the script. -TestLink scans test result XMLs from Bazel (bazel-testlogs) or in the folder 'tests-report' and converts each test case with metadata into Sphinx external needs, allowing links from tests to requirements. +Combo build: + - `repo_name`: repository name (parsed from filepath in step 1) + - `hash`: revision/commit hash (as provided by the known_good_json) + - `url`: repository remote URL (as provided by the known_good_json) + +Local build: + - `repo_name`: 'local_repo' + - `hash`: will be empty at this point, and later filled via parsing the git commands + - `url`: will be empty at this point, and later filled via parsing the git commands + +--- + +### Source Code Linker Extension + +1a. **Reads the merged JSON** produced by Bazel +1b. **Reads test.xml files and generates testlinks JSON** +2. **Validates and adapts** both JSON cache files +3. **Merges and converts** them into the final structure: **RepoSourceLink** + +#### Codelinks + +Since these are all found in [Bazel step 1](#step-1-per-repository-cache-generation) this essentially does not run +anymore inside of the source_code_linker extension. + +#### Testlinks + +TestLink scans test result XMLs from Bazel (bazel-testlogs) or from the folder 'tests-report' and converts each test case with metadata into Sphinx external needs, allowing links from tests to requirements. This depends on the `attribute_plugin` in our tooling repository, find it [here](https://github.com/eclipse-score/tooling/tree/main/python_basics/score_pytest) + +:::attention +If TestLinks should be generated in a combo build please ensure that you have the known_good_json added to the docs macro. +::: + #### Test Tagging Options ```python @@ -90,7 +145,7 @@ def test_feature(): #### Data Flow 1. **XML Parsing** (`xml_parser.py`) - - Scans `bazel-testlogs/` for `test.xml` files. + - Scans `bazel-testlogs/` or `tests-report` for `test.xml` files. - Parses test cases and extracts: - Name & Classname - File path @@ -98,6 +153,7 @@ def test_feature(): - Result (e.g. passed, failed, skipped) - Result text (if failed/skipped will check if message was attached to it) - Verifications (`PartiallyVerifies`, `FullyVerifies`) + - Also parses metadata according to the [metadata rules](#repo-metadata-rules) - Cases without metadata are logged out as info (not errors). - Test cases with metadata are converted into: @@ -125,17 +181,23 @@ The DataFromTestCase depicts the information gathered about one testcase. "result_text": "", "PartiallyVerifies": "tool_req__docs_common_attr_title, tool_req__docs_common_attr_description", "FullyVerifies": null + "repo_name": "local_module", + "hash": "", + "url": "", } ] ``` --- -## ๐Ÿ”— Combined Links -During the Sphinx build process, both CodeLink and TestLink data are combined and applied to needs. +### Early-Combined Links -This is handled in `__init__.py` using the `NeedSourceLinks` and `SourceCodeLinks` dataclasses from `need_source_links.py`. +During the Sphinx build process, both CodeLink and TestLink data are combined and grouped by needs. +This will allow us to have an easier time building the final JSON which will eliminate the metadata from each link, +and group all needs according to the appropriate repository. + +This all is handled in `need_source_links.py`. ### Combined JSON Example @@ -150,7 +212,11 @@ This is handled in `__init__.py` using the `NeedSourceLinks` and `SourceCodeLink "line": 33, "tag": "#--req-Id:",# added `--` to avoid detection "need": "tool_req__docs_common_attr_title", - "full_line": "#--req-Id: tool_req__docs_common_attr_title" # added `--` to avoid detection + "full_line": "#--req-Id: tool_req__docs_common_attr_title", # added `--` to avoid detection + "repo_name": "local_module", + "hash": "", + "url": "", + } ], "TestLinks": [ @@ -161,7 +227,10 @@ This is handled in `__init__.py` using the `NeedSourceLinks` and `SourceCodeLink "need": "tool_req__docs_common_attr_title", "verify_type": "partially", "result": "passed", - "result_text": "" + "result_text": "", + "repo_name": "local_module", + "hash": "", + "url": "", } ] } @@ -169,36 +238,116 @@ This is handled in `__init__.py` using the `NeedSourceLinks` and `SourceCodeLink ] ``` +#### Final layout: RepoSourceLink + +Instead of repeating `repo_name/hash/url` for every single link entry, the final output groups links **by repository**: + +- Repository metadata appears **once per repository** +- All links belonging to that repository are stored beneath it + +This somewhat looks like this: +```{code-block} json +[ + { + "repo": { + "name": "local_repo", + "hash": "", + "url": "" + }, + "needs": [ + { + "need": "tool_req__docs_common_attr_id_scheme", + "links": { + "CodeLinks": [ + { + "file": "src/extensions/score_metamodel/checks/attributes_format.py", + "line": 30, + "tag": "# req-Id:", + "need": "tool_req__docs_common_attr_id_scheme", + "full_line": "# req-Id: tool_req__docs_common_attr_id_scheme" + } + ], + "TestLinks": [] + } + }, + ], +``` +Due to not saving the `repo_name, url and hash` in each link but grouping them we can eleminate a lot of unnecessary +length to the JSON document here. + +This is the structure the extension uses to attach links to needs (via the `source_code_link` attribute), while keeping the resulting data compact and normalized. + +:::hint +Currently if the repo is the local one, SCL is still using the hash and url it gets from executing the git functions parsing. +This is a known inefficiency and will be improved upon later +::: --- -## โš ๏ธ Known Limitations +## Result: Traceability Links in Needs + +During the Sphinx build process, the extension applies the computed links to needs: -### CodeLink +- Each needโ€™s `source_code_link` and `testlink` attribute is filled from the (repo-grouped) RepoSourceLink data where applicable. +- If a referenced need ID does not exist, a build warning will be raised. + +--- -- โŒ Not compatible with **Esbonio/Live_preview** -- ๐Ÿ”— GitHub links may 404 if the commit isnโ€™t pushed -- ๐Ÿงช Tags must match exactly (e.g. # req-Id) -- ๐Ÿ‘€ `source_code_link` isnโ€™t visible until the full Sphinx build is completed +## Known Limitations + +### General +- In combo builds, known_good_json is required. +- inefficiencies in creating the links and saving the JSON caches +- Not compatible with **Esbonio/Live_preview** + +### Codelinks +- GitHub links may 404 if the commit isnโ€™t pushed +- Tags must match exactly (e.g. # req-Id) +- `source_code_link` isnโ€™t visible until the full Sphinx build is completed ### TestLink -- โŒ Not compatible with **Esbonio/Live_preview** -- ๐Ÿ”— GitHub links may 404 if the commit isnโ€™t pushed -- ๐Ÿงช XML structure must be followed exactly (e.g. `properties & attributes`) -- ๐Ÿ—‚ Relies on test to be executed first +- GitHub links may 404 if the commit isnโ€™t pushed +- XML structure must be followed exactly (e.g. `properties & attributes`) +- Relies on test to be executed first + +--- +## High-level Data Flow Summary +1. **Bazel Script #1**: scan + parse โ†’ write **per-repo cache** (includes repo metadata if known) +2. **Bazel Script #2**: merge caches โ†’ write **single merged JSON** +3. **Sphinx Extension**: read merged JSON โ†’ adapt to **RepoSourceLink** โ†’ inject source_code_link and testlink into needs --- -## ๐Ÿ—๏ธ Internal Module Overview +## Clearing Cache Manually + +To clear the build cache, run: + +```bash +rm -rf _build/ +``` + +## Internal Overview +The bazel part: +``` +scripts_bazel/ +โ”œโ”€โ”€ BUILD # Declare libraries and filegroups needed for bazel +โ”œโ”€โ”€ generate_sourcelinks_cli.py # Bazel step 1 => Parses sourcefiles for tags +โ”œโ”€โ”€ merge_sourcelinks.py +โ””โ”€โ”€ tests +โ”‚ โ””โ”€โ”€ ... +``` +The Sphinx extension ``` score_source_code_linker/ โ”œโ”€โ”€ __init__.py # Main Sphinx extension; combines CodeLinks + TestLinks -โ”œโ”€โ”€ generate_source_code_links_json.py # Parses source files for tags +โ”œโ”€โ”€ generate_source_code_links_json.py # Most functionality moved to 'scripts_bazel/generate_sourcelinks_cli' โ”œโ”€โ”€ need_source_links.py # Data model for combined links +โ”œโ”€โ”€ repo_source_links.py # Data model for Repo combined links (Final output JSON) +โ”œโ”€โ”€ helpers.py # Misc. functions used throughout SCL โ”œโ”€โ”€ needlinks.py # CodeLink dataclass & JSON encoder/decoder โ”œโ”€โ”€ testlink.py # DataForTestLink definition & logic โ”œโ”€โ”€ xml_parser.py # Parses XML files into test case data @@ -222,81 +371,3 @@ To see working examples for CodeLinks & TestLinks, take a look at the Docs-As-Co [Example CodeLink](https://eclipse-score.github.io/docs-as-code/main/requirements/requirements.html#tool_req__docs_common_attr_status) [Example TestLink](https://eclipse-score.github.io/docs-as-code/main/requirements/requirements.html#tool_req__docs_dd_link_source_code_link) - -## Flow-Overview -```{mermaid} -flowchart TD - %% Entry Point - A[source_code_linker] --> B{Check for Grouped JSON Cache} - - %% If cache exists - B --> |โœ…| C[Load Grouped JSON Cache] - B --> |๐Ÿ”ด| N9[Proceed Without Cache] - - %% --- NeedLink Path --- - N9 --> D1[needslink.py
NeedLink] - D1 --> E1{Check for CodeLink JSON Cache} - - E1 --> |โœ…| F1[Load CodeLink JSON Cache] - F1 --> Z[Grouped JSON Cache] - - E1 --> |๐Ÿ”ด| G1[Parse all files in repository] - G1 --> H1[Build & Save
CodeLink JSON Cache] - H1 --> Z - - %% --- TestLink Path --- - N9 --> D2[testlink.py
DFTL] - D2 --> E2{Check for DFTL JSON Cache} - - E2 --> |โœ…| F2[Load DFTL JSON Cache] - F2 --> J2[Load DOTC JSON Cache] - J2 --> K2[Add as External Needs] - - E2 --> |๐Ÿ”ด| G2[Parse test.xml Files] - G2 --> H2[Convert TestCases
to DOTC] - H2 --> I2[Build & Save
DOTC JSON Cache] - I2 --> K2 - - H2 --> M2[Convert to DFTL] - M2 --> N2[Build & Save
DFTL JSON Cache] - N2 --> Z - - %% Final step - Z --> FINAL[Add links to needs] - - %% Legend - subgraph Legend["Legend"] - direction TB - L1[NeedLink Operations] - L2[TestLink Operations] - L4[DTFL = DataForTestLink] - L3[TestCaseNeed Operations] - L5[DOTC = DataOfTestCase] - L1 ~~~ L2 - L2 ~~~ L4 - L4 ~~~ L3 - L3 ~~~ L5 - end - - %% Node Styling - classDef needlink fill:#3b82f6,color:#ffffff,stroke:#1d4ed8,stroke-width:2px - classDef testlink fill:#8b5cf6,color:#ffffff,stroke:#6d28d9,stroke-width:2px - classDef dotc fill:#f59e0b,color:#ffffff,stroke:#b45309,stroke-width:2px - classDef grouped fill:#10b981,color:#ffffff,stroke:#047857,stroke-width:2px - classDef final fill:#f43f5e,color:#ffffff,stroke:#be123c,stroke-width:2px - - %% Class assignments - class D1,E1,F1,G1,H1 needlink - class D2,E2,F2,G2,M2,N2 testlink - class J2,H2,I2,K2 dotc - class Z grouped - class FINAL final - class L1 needlink - class L2,L4 testlink - class L3,L5 dotc - - %% Edge/Arrow Styling - linkStyle default stroke:#22d3ee,stroke-width:2px,color:#22d3ee - %% Ensure links in the Legend do not show up - linkStyle 23,24,25,26 opacity:0 -``` diff --git a/src/extensions/score_draw_uml_funcs/__init__.py b/src/extensions/score_draw_uml_funcs/__init__.py index d4a95f39d..3388463b7 100644 --- a/src/extensions/score_draw_uml_funcs/__init__.py +++ b/src/extensions/score_draw_uml_funcs/__init__.py @@ -32,7 +32,7 @@ from pathlib import Path from typing import Any, cast -from score_draw_uml_funcs.helpers import ( +from score_draw_uml_funcs.draw_helpers import ( gen_header, gen_interface_element, gen_link_text, diff --git a/src/extensions/score_draw_uml_funcs/helpers.py b/src/extensions/score_draw_uml_funcs/draw_helpers.py similarity index 100% rename from src/extensions/score_draw_uml_funcs/helpers.py rename to src/extensions/score_draw_uml_funcs/draw_helpers.py diff --git a/src/extensions/score_source_code_linker/BUILD b/src/extensions/score_source_code_linker/BUILD index e3c289c66..e9e3528f7 100644 --- a/src/extensions/score_source_code_linker/BUILD +++ b/src/extensions/score_source_code_linker/BUILD @@ -54,6 +54,8 @@ py_library( "needlinks.py", "testlink.py", "xml_parser.py", + "helpers.py", + "repo_source_links.py", ], imports = ["."], visibility = ["//visibility:public"], diff --git a/src/extensions/score_source_code_linker/__init__.py b/src/extensions/score_source_code_linker/__init__.py index cf9843dc5..29628b5ca 100644 --- a/src/extensions/score_source_code_linker/__init__.py +++ b/src/extensions/score_source_code_linker/__init__.py @@ -21,7 +21,6 @@ # This whole directory implements the above mentioned tool requirements import os -from collections import defaultdict from copy import deepcopy from pathlib import Path from typing import cast @@ -35,18 +34,22 @@ from src.extensions.score_source_code_linker.generate_source_code_links_json import ( generate_source_code_links_json, ) +from src.extensions.score_source_code_linker.helpers import get_github_link from src.extensions.score_source_code_linker.need_source_links import ( - NeedSourceLinks, - SourceCodeLinks, + group_by_need, load_source_code_links_combined_json, store_source_code_links_combined_json, ) from src.extensions.score_source_code_linker.needlinks import ( - NeedLink, load_source_code_links_json, + load_source_code_links_with_metadata_json, +) +from src.extensions.score_source_code_linker.repo_source_links import ( + group_needs_by_repo, + load_repo_source_links_json, + store_repo_source_links_json, ) from src.extensions.score_source_code_linker.testlink import ( - DataForTestLink, load_data_of_test_case_json, load_test_xml_parsed_json, ) @@ -58,7 +61,6 @@ find_git_root, find_ws_root, ) -from src.helper_lib.additional_functions import get_github_link LOGGER = get_logger(__name__) # Uncomment this to enable more verbose logging @@ -71,53 +73,6 @@ # โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ -def group_by_need( - source_code_links: list[NeedLink], - test_case_links: list[DataForTestLink] | None = None, -) -> list[SourceCodeLinks]: - """ - Groups the given need links and test case links by their need ID. - Returns a nested dictionary structure with 'CodeLink' and 'TestLink' categories. - Example output: - - - { - "need": "", - "links": { - "CodeLinks": [NeedLink, NeedLink, ...], - "TestLinks": [testlink, testlink, ...] - } - } - """ - # TODO: I wonder if there is a more efficent way to do this - grouped_by_need: dict[str, NeedSourceLinks] = defaultdict( - lambda: NeedSourceLinks(TestLinks=[], CodeLinks=[]) - ) - - # Group source code links - for needlink in source_code_links: - grouped_by_need[needlink.need].CodeLinks.append(needlink) - - # Group test case links - if test_case_links is not None: - for testlink in test_case_links: - grouped_by_need[testlink.need].TestLinks.append(testlink) - - # Build final list of SourceCodeLinks - result: list[SourceCodeLinks] = [ - SourceCodeLinks( - need=need, - links=NeedSourceLinks( - CodeLinks=need_links.CodeLinks, - TestLinks=need_links.TestLinks, - ), - ) - for need, need_links in grouped_by_need.items() - ] - - return result - - def get_cache_filename(build_dir: Path, filename: str) -> Path: """ Returns the path to the cache file for the source code linker. @@ -142,14 +97,19 @@ def build_and_save_combined_file(outdir: Path): else: source_code_links_json = Path(source_code_links_json) - source_code_links = load_source_code_links_json(source_code_links_json) + # This isn't pretty will think of a better solution later, for now this should work + try: + source_code_links = load_source_code_links_json(source_code_links_json) + except AssertionError: + source_code_links = load_source_code_links_with_metadata_json( + source_code_links_json + ) test_code_links = load_test_xml_parsed_json( get_cache_filename(outdir, "score_xml_parser_cache.json") ) - + scl_list = group_by_need(source_code_links, test_code_links) store_source_code_links_combined_json( - outdir / "score_scl_grouped_cache.json", - group_by_need(source_code_links, test_code_links), + outdir / "score_scl_grouped_cache.json", scl_list ) @@ -254,17 +214,20 @@ def setup_test_code_linker(app: Sphinx, env: BuildEnvironment): def register_combined_linker(app: Sphinx): - # Registering the combined linker to Sphinx + # Registering the final combine linker to Sphinx # priority is set to make sure it is called in the right order. - # Needs to be called after xml parsing & codelink - app.connect("env-updated", setup_combined_linker, priority=507) + # Needs to be called after xml parsing & codelink & combined_linker + app.connect("env-updated", setup_combined_linker, priority=510) def setup_combined_linker(app: Sphinx, _: BuildEnvironment): grouped_cache = get_cache_filename(app.outdir, "score_scl_grouped_cache.json") - gruped_cache_exists = grouped_cache.exists() + grouped_cache_exists = grouped_cache.exists() # TODO this cache should be done via Bazel - if not gruped_cache_exists or not app.config.skip_rescanning_via_source_code_linker: + if ( + not grouped_cache_exists + or not app.config.skip_rescanning_via_source_code_linker + ): LOGGER.debug( "Did not find combined json 'score_scl_grouped_cache.json' in _build." "Generating new one" @@ -272,6 +235,40 @@ def setup_combined_linker(app: Sphinx, _: BuildEnvironment): build_and_save_combined_file(app.outdir) +def register_repo_linker(app: Sphinx): + # Registering the combined linker to Sphinx + # priority is set to make sure it is called in the right order. + # Needs to be called after xml parsing & codelink + app.connect("env-updated", setup_repo_linker, priority=520) + + +def build_and_save_repo_scl_file(outdir: Path): + scl_links = load_source_code_links_combined_json( + get_cache_filename(outdir, "score_scl_grouped_cache.json") + ) + mcl_links = group_needs_by_repo(scl_links) + store_repo_source_links_json( + outdir / "score_repo_grouped_scl_cache.json", mcl_links + ) + + +def setup_repo_linker(app: Sphinx, _: BuildEnvironment): + grouped_cache = get_cache_filename( + app.outdir, "score_repo_grouped_scl_cache.json" + ) + grouped_cache_exists = grouped_cache.exists() + # TODO this cache should be done via Bazel + if ( + not grouped_cache_exists + or not app.config.skip_rescanning_via_source_code_linker + ): + LOGGER.debug( + "Did not find combined json 'score_module_grouped_scl_cache.json' " + "in _build. Generating new one" + ) + build_and_save_repo_scl_file(app.outdir) + + def setup_once(app: Sphinx): # might be the only way to solve this? if "skip_rescanning_via_source_code_linker" in app.config: @@ -295,14 +292,16 @@ def setup_once(app: Sphinx): setup_source_code_linker(app, ws_root) register_test_code_linker(app) register_combined_linker(app) + register_repo_linker(app) - # Priorty=510 to ensure it's called after the test code linker & combined connection - app.connect("env-updated", inject_links_into_needs, priority=510) + # Priority=515 to ensure it's called after the test linker & combined connection + app.connect("env-updated", inject_links_into_needs, priority=525) def setup(app: Sphinx) -> dict[str, str | bool]: # Esbonio will execute setup() on every iteration. # setup_once will only be called once. + app.add_config_value("KNOWN_GOOD_JSON", default="",rebuild="env",types=str) setup_once(app) return { @@ -352,42 +351,43 @@ def inject_links_into_needs(app: Sphinx, env: BuildEnvironment) -> None: f"?? Need {id} already has testlink: {need.get('testlink')}" ) - source_code_links_by_need = load_source_code_links_combined_json( - get_cache_filename(app.outdir, "score_scl_grouped_cache.json") + scl_by_module = load_repo_source_links_json( + get_cache_filename(app.outdir, "score_repo_grouped_scl_cache.json") ) + for module_grouped_needs in scl_by_module: + for source_code_links in module_grouped_needs.needs: + need = find_need(needs_copy, source_code_links.need) + if need is None: + # TODO: print github annotations as in https://github.com/eclipse-score/bazel_registry/blob/7423b9996a45dd0a9ec868e06a970330ee71cf4f/tools/verify_semver_compatibility_level.py#L126-L129 + for n in source_code_links.links.CodeLinks: + LOGGER.warning( + f"{n.file}:{n.line}: Could not find {source_code_links.need} " + "in documentation [CODE LINK]", + type="score_source_code_linker", + ) + for n in source_code_links.links.TestLinks: + LOGGER.warning( + f"{n.file}:{n.line}: Could not find {source_code_links.need} " + "in documentation [TEST LINK]", + type="score_source_code_linker", + ) + continue + + need_as_dict = cast(dict[str, object], need) + metadata = module_grouped_needs.repo + need_as_dict["source_code_link"] = ", ".join( + f"{get_github_link(metadata, n)}<>{n.file}:{n.line}" + for n in source_code_links.links.CodeLinks + ) + need_as_dict["testlink"] = ", ".join( + f"{get_github_link(metadata, n)}<>{n.name}" + for n in source_code_links.links.TestLinks + ) - for source_code_links in source_code_links_by_need: - need = find_need(needs_copy, source_code_links.need) - if need is None: - # TODO: print github annotations as in https://github.com/eclipse-score/bazel_registry/blob/7423b9996a45dd0a9ec868e06a970330ee71cf4f/tools/verify_semver_compatibility_level.py#L126-L129 - for n in source_code_links.links.CodeLinks: - LOGGER.warning( - f"{n.file}:{n.line}: Could not find {source_code_links.need} " - "in documentation [CODE LINK]", - type="score_source_code_linker", - ) - for n in source_code_links.links.TestLinks: - LOGGER.warning( - f"{n.file}:{n.line}: Could not find {source_code_links.need} " - "in documentation [TEST LINK]", - type="score_source_code_linker", - ) - continue - - need_as_dict = cast(dict[str, object], need) - - need_as_dict["source_code_link"] = ", ".join( - f"{get_github_link(n)}<>{n.file}:{n.line}" - for n in source_code_links.links.CodeLinks - ) - need_as_dict["testlink"] = ", ".join( - f"{get_github_link(n)}<>{n.name}" for n in source_code_links.links.TestLinks - ) - - # NOTE: Removing & adding the need is important to make sure - # the needs gets 're-evaluated'. - Needs_Data.remove_need(need["id"]) - Needs_Data.add_need(need) + # NOTE: Removing & adding the need is important to make sure + # the needs gets 're-evaluated'. + Needs_Data.remove_need(need["id"]) + Needs_Data.add_need(need) # โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ diff --git a/src/extensions/score_source_code_linker/generate_source_code_links_json.py b/src/extensions/score_source_code_linker/generate_source_code_links_json.py index abedc2db4..7c48de9ee 100644 --- a/src/extensions/score_source_code_linker/generate_source_code_links_json.py +++ b/src/extensions/score_source_code_linker/generate_source_code_links_json.py @@ -20,11 +20,15 @@ import os from pathlib import Path +from sphinx_needs.logging import get_logger + from src.extensions.score_source_code_linker.needlinks import ( NeedLink, store_source_code_links_json, ) +LOGGER = get_logger(__name__) + TAGS = [ "# " + "req-traceability:", "# " + "req-Id:", @@ -43,21 +47,29 @@ def _extract_references_from_line(line: str): yield tag, req.strip() -def _extract_references_from_file(root: Path, file_path: Path) -> list[NeedLink]: - """Scan a single file for template strings and return findings.""" +def _extract_references_from_file( + root: Path, file_path_name: Path, file_path: Path +) -> list[NeedLink]: + """Scan a single file for template strings and return findings. + Examples: + # ROOT: /docs-as-code/src/extensions/score_source_code_linker + #FILE PATH: + external/score_docs_as_code+/src/extensions/score_source_code_linker/testlink.py + #FILE PATH NAME: testlink.py + """ assert root.is_absolute(), "Root path must be absolute" - assert not file_path.is_absolute(), "File path must be relative to the root" + assert not file_path_name.is_absolute(), "File path must be relative to the root" # assert file_path.is_relative_to(root), ( # f"File path ({file_path}) must be relative to the root ({root})" # ) - assert (root / file_path).exists(), ( - f"File {file_path} does not exist in root {root}." + assert (root / file_path_name).exists(), ( + f"File {file_path_name} does not exist in root {root}." ) findings: list[NeedLink] = [] try: - with open(root / file_path, encoding="utf-8", errors="ignore") as f: + with open(root / file_path_name, encoding="utf-8", errors="ignore") as f: for line_num, line in enumerate(f, 1): for tag, req in _extract_references_from_line(line): findings.append( @@ -69,8 +81,9 @@ def _extract_references_from_file(root: Path, file_path: Path) -> list[NeedLink] full_line=line.strip(), ) ) - except (UnicodeDecodeError, PermissionError, OSError): + except (UnicodeDecodeError, PermissionError, OSError) as e: # Skip files that can't be read as text + LOGGER.debug(f"Error reading file to parse for linked needs: \n{e}") pass return findings @@ -117,12 +130,19 @@ def find_all_need_references(search_path: Path) -> list[NeedLink]: # Use os.walk to have better control over directory traversal for file in iterate_files_recursively(search_path): - references = _extract_references_from_file(search_path, file) + LOGGER.debug( + f"Scanning file by the name of: {file.name} " + f"in path: {search_path} with the file being: {file}" + ) + # print("Search_path: ", search_path) + # print("File.name: ", file.name) + # print("File: ", file) + references = _extract_references_from_file(search_path, Path(file), file) all_need_references.extend(references) elapsed_time = os.times().elapsed - start_time - print( - f"DEBUG: Found {len(all_need_references)} need references " + LOGGER.debug( + f"Found {len(all_need_references)} need references " f"in {elapsed_time:.2f} seconds" ) diff --git a/src/extensions/score_source_code_linker/helpers.py b/src/extensions/score_source_code_linker/helpers.py new file mode 100644 index 000000000..5311717eb --- /dev/null +++ b/src/extensions/score_source_code_linker/helpers.py @@ -0,0 +1,111 @@ +# ******************************************************************************* +# Copyright (c) 2025 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import json +from pathlib import Path + +# Import types that depend on score_source_code_linker +from src.extensions.score_source_code_linker.needlinks import DefaultNeedLink, NeedLink +from src.extensions.score_source_code_linker.repo_source_links import RepoInfo +from src.extensions.score_source_code_linker.testlink import ( + DataForTestLink, + DataOfTestCase, +) +from src.helper_lib import ( + find_git_root, + get_current_git_hash, + get_github_base_url, +) + + +def get_github_link( + metadata: RepoInfo, + link: NeedLink | DataForTestLink | DataOfTestCase | None = None, +) -> str: + if link is None: + link = DefaultNeedLink() + if not metadata.hash: + # Local path (//:docs) + return get_github_link_from_git(link) + # Ref-Integration path (//:docs_combo..) + return get_github_link_from_json(metadata, link) + + +def get_github_link_from_git( + link: NeedLink | DataForTestLink | DataOfTestCase | None = None, +) -> str: + if link is None: + link = DefaultNeedLink() + passed_git_root = find_git_root() + if passed_git_root is None: + passed_git_root = Path() + base_url = get_github_base_url() + current_hash = get_current_git_hash(passed_git_root) + return f"{base_url}/blob/{current_hash}/{link.file}#L{link.line}" + + +def get_github_link_from_json( + metadata: RepoInfo, + link: NeedLink | DataForTestLink | DataOfTestCase | None = None, +) -> str: + if link is None: + link = DefaultNeedLink() + base_url = metadata.url + current_hash = metadata.hash + return f"{base_url}/blob/{current_hash}/{link.file}#L{link.line}" + + +def parse_repo_name_from_path(path: Path) -> str: + """ + Parse out the Module-Name from the filename: + Combo Example: + Path: external/score_docs_as_code+/src/helper_lib/test_helper_lib.py + => score_docs_as_code + Local: + Path: src/helper_lib/test_helper_lib.py + => local_repo + + """ + + # COMBO BUILD + + if str(path).startswith("external/"): + # This allows for files / folders etc. to have `external` in their name too. + module_raw = str(path).removeprefix("external/") + filepath_split = str(module_raw).split("/", maxsplit=1) + return str(filepath_split[0].removesuffix("+")) + # We return this when we are in a local build `//:docs` the rest of DaC knows + return "local_repo" + + +def parse_info_from_known_good( + known_good_json: Path, repo_name: str +) -> tuple[str, str]: + with open(known_good_json) as f: + kg_json = json.load(f) + + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€[ Assert our worldview that has to exist here ]โ”€โ”€โ”€โ”€โ”€ + assert kg_json, ( + f"Known good json at: {known_good_json} is empty. This is not allowed" + ) + assert "modules" in kg_json, ( + f"Known good json at: {known_good_json} is missing the 'modules' key" + ) + assert kg_json["modules"], ( + f"Known good json at: {known_good_json} has an empty 'modules' dictionary" + ) + + for category in kg_json["modules"].values(): + if repo_name in category: + m = category[repo_name] + return (m["hash"], m["repo"].removesuffix(".git")) + raise KeyError(f"Module {repo_name} not found in known_good_json.") diff --git a/src/extensions/score_source_code_linker/need_source_links.py b/src/extensions/score_source_code_linker/need_source_links.py index 6c738da8e..9a5d3c29e 100644 --- a/src/extensions/score_source_code_linker/need_source_links.py +++ b/src/extensions/score_source_code_linker/need_source_links.py @@ -20,6 +20,7 @@ # req-Id: tool_req__docs_dd_link_source_code_link import json +from collections import defaultdict from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any @@ -81,9 +82,9 @@ def SourceCodeLinks_JSON_Decoder(d: dict[str, Any]) -> SourceCodeLinks | dict[st def store_source_code_links_combined_json( file: Path, source_code_links: list[SourceCodeLinks] ): - # After `rm -rf _build` or on clean builds the directory does not exist, so we need - # to create it - file.parent.mkdir(exist_ok=True) + # After `rm -rf _build` or on clean builds the directory does not exist, + # so we need to create it. We create any folder that might be missing + file.parent.mkdir(exist_ok=True, parents=True) with open(file, "w") as f: json.dump( source_code_links, @@ -108,3 +109,50 @@ def load_source_code_links_combined_json(file: Path) -> list[SourceCodeLinks]: "SourceCodeLinks objects." ) return links + + +def group_by_need( + source_code_links: list[NeedLink], + test_case_links: list[DataForTestLink] | None = None, +) -> list[SourceCodeLinks]: + """ + Groups the given need links and test case links by their need ID. + Returns a nested dictionary structure with 'CodeLink' and 'TestLink' categories. + Example output: + + + { + "need": "", + "links": { + "CodeLinks": [NeedLink, NeedLink, ...], + "TestLinks": [testlink, testlink, ...] + } + } + """ + # TODO: I wonder if there is a more efficent way to do this + grouped_by_need: dict[str, NeedSourceLinks] = defaultdict( + lambda: NeedSourceLinks(TestLinks=[], CodeLinks=[]) + ) + + # Group source code links + for needlink in source_code_links: + grouped_by_need[needlink.need].CodeLinks.append(needlink) + + # Group test case links + if test_case_links is not None: + for testlink in test_case_links: + grouped_by_need[testlink.need].TestLinks.append(testlink) + + # Build final list of SourceCodeLinks + result: list[SourceCodeLinks] = [ + SourceCodeLinks( + need=need, + links=NeedSourceLinks( + CodeLinks=need_links.CodeLinks, + TestLinks=need_links.TestLinks, + ), + ) + for need, need_links in grouped_by_need.items() + ] + + return result diff --git a/src/extensions/score_source_code_linker/needlinks.py b/src/extensions/score_source_code_linker/needlinks.py index 348147292..fb099e14a 100644 --- a/src/extensions/score_source_code_linker/needlinks.py +++ b/src/extensions/score_source_code_linker/needlinks.py @@ -16,10 +16,26 @@ import os from dataclasses import asdict, dataclass from pathlib import Path -from typing import Any +from typing import Any, TypedDict, TypeGuard -@dataclass(frozen=True, order=True) +class MetaData(TypedDict): + repo_name: str + hash: str + url: str + + +def DefaultMetaData() -> MetaData: + md: MetaData = {"repo_name": "local_repo", "hash": "", "url": ""} + return md + + +def is_metadata(x: object) -> TypeGuard[MetaData]: + # Make this as strict/loose as you want; at minimum, it must be a dict. + return isinstance(x, dict) and {"repo_name", "hash", "url"} <= x.keys() + + +@dataclass(order=True) class NeedLink: """Represents a single template string finding in a file.""" @@ -28,6 +44,53 @@ class NeedLink: tag: str need: str full_line: str + repo_name: str = "local_repo" + hash: str = "" + url: str = "" + + # Adding hashing & equality as this is needed to make comparisions. + # Since the Dataclass is not 'frozen = true' it isn't automatically hashable + # This is used in tests + def __hash__(self): + return hash( + ( + self.file, + self.line, + self.tag, + self.need, + self.full_line, + self.repo_name, + self.hash, + self.url, + ) + ) + + def __eq__(self, other: Any): + if not isinstance(other, NeedLink): + return NotImplemented + return ( + self.file == other.file + and self.line == other.line + and self.tag == other.tag + and self.need == other.need + and self.full_line == other.full_line + and self.repo_name == other.repo_name + and self.hash == other.hash + and self.url == other.url + ) + + # Normal 'dictionary conversion'. Converts all fields + def to_dict_full(self) -> dict[str, str | Path | int]: + return asdict(self) + + # Drops MetaData fields for saving the Dataclass (saving space in json) + # The information is in the 'Repo_Source_Link' in the end + def to_dict_without_metadata(self) -> dict[str, str | Path | int]: + d = asdict(self) + d.pop("repo_name", None) + d.pop("hash", None) + d.pop("url", None) + return d def DefaultNeedLink() -> NeedLink: @@ -41,6 +104,8 @@ def DefaultNeedLink() -> NeedLink: tag="", need="", full_line="", + # Repo_name, hash, url are defaulted to "" + # therefore not needed to be listed ) @@ -61,26 +126,100 @@ def needlink_decoder(d: dict[str, Any]) -> NeedLink | dict[str, Any]: tag=d["tag"], need=d["need"], full_line=d["full_line"], + repo_name=d.get("repo_name", ""), + hash=d.get("hash", ""), + url=d.get("url", ""), ) # It's something else, pass it on to other decoders return d -def store_source_code_links_json(file: Path, needlist: list[NeedLink]): +def store_source_code_links_with_metadata_json( + file: Path, metadata: MetaData, needlist: list[NeedLink] +) -> None: + """ + Writes a JSON array: + [ meta_dict, needlink1, needlink2, ... ] + + meta_dict must include: + repo_name, hash, url + """ + payload: list[object] = [metadata, *needlist] + + # After `rm -rf _build` or on clean builds the directory does not exist, + # so we need to create it. We create any folder that might be missing + file.parent.mkdir(exist_ok=True, parents=True) + with open(file, "w", encoding="utf-8") as f: + json.dump(payload, f, cls=NeedLinkEncoder, indent=2, ensure_ascii=False) + + +def store_source_code_links_json(file: Path, needlist: list[NeedLink]) -> None: + """ + Writes a JSON array: + [ needlink1, needlink2, ... ] + """ + # After `rm -rf _build` or on clean builds the directory does not exist, - # so we need to create it - file.parent.mkdir(exist_ok=True) - with open(file, "w") as f: - json.dump( - needlist, - f, - cls=NeedLinkEncoder, # use your custom encoder - indent=2, - ensure_ascii=False, + # so we need to create it. We create any folder that might be missing + file.parent.mkdir(exist_ok=True, parents=True) + with open(file, "w", encoding="utf-8") as f: + json.dump(needlist, f, cls=NeedLinkEncoder, indent=2, ensure_ascii=False) + + +def _is_needlink_list(xs: list[object]) -> TypeGuard[list[NeedLink]]: + return all(isinstance(link, NeedLink) for link in xs) + + +def load_source_code_links_with_metadata_json(file: Path) -> list[NeedLink]: + """ + Expects the JSON array where first is a meta_dict: + [ meta_dict, needlink1, needlink2, ... ] + Returns: + [NeedLink, NeedLink, ...] + + This normally should be the one called 'locally' => :docs target + """ + if not file.is_absolute(): + ws_root = os.environ.get("BUILD_WORKSPACE_DIRECTORY") + if ws_root: + file = Path(ws_root) / file + + data: list[object] = json.loads( + file.read_text(encoding="utf-8"), + object_hook=needlink_decoder, + ) + links: list[object] = [] + if not is_metadata(data[0]): + raise TypeError( + "If you do not have a 'metadata' dict as the first one in the json " + "you might wanted to call the load without metadata named: " + "'load_source_code_links_json'" ) + metadata: MetaData = data[0] + links = data[1:] + if not _is_needlink_list(links): + raise TypeError( + "In local build context all items after" + f"metadata must decode to NeedLink objects. File: {file}" + ) + for d in links: + d.repo_name = metadata["repo_name"] + d.hash = metadata["hash"] + d.url = metadata["url"] + return links def load_source_code_links_json(file: Path) -> list[NeedLink]: + """ + Expects the JSON array with needlinks + *that already have extra info in them* (repo_name, hash, url): + [ needlink1, needlink2, ... ] + Returns: + [NeedLink, NeedLink, ...] + + This normally should be the one called in combo builds + => :docs_combo_experimental target + """ if not file.is_absolute(): # use env variable set by Bazel ws_root = os.environ.get("BUILD_WORKSPACE_DIRECTORY") diff --git a/src/extensions/score_source_code_linker/repo_source_links.py b/src/extensions/score_source_code_linker/repo_source_links.py new file mode 100644 index 000000000..4fbd2adaa --- /dev/null +++ b/src/extensions/score_source_code_linker/repo_source_links.py @@ -0,0 +1,154 @@ +# ******************************************************************************* +# Copyright (c) 2025 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + + +import json +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + +from src.extensions.score_source_code_linker.need_source_links import ( + NeedSourceLinks, + SourceCodeLinks, + SourceCodeLinks_JSON_Decoder, +) +from src.extensions.score_source_code_linker.needlinks import NeedLink +from src.extensions.score_source_code_linker.testlink import DataForTestLink + + +@dataclass +class RepoInfo: + name: str + hash: str + url: str + + +def DefaultRepoInfo() -> RepoInfo: + return RepoInfo(name="local_repo", hash="", url="") + + +@dataclass +class RepoSourceLinks: + repo: RepoInfo + needs: list[SourceCodeLinks] = field(default_factory=list) + + +class RepoSourceLinks_TEST_JSON_Encoder(json.JSONEncoder): + def default(self, o: object) -> str | dict[str, Any]: + if isinstance(o, Path): + return str(o) + # We do not want to save the metadata inside the codelink or testlink + # As we save this already in a structure above it + # (hash, module_name, url) + if isinstance(o, NeedLink | DataForTestLink): + return o.to_dict_without_metadata() + # We need to split this up, otherwise the nested + # dictionaries won't get split up and we will not + # run into the 'to_dict_without_metadata' as + # everything will be converted to a normal dictionary + if isinstance(o, RepoSourceLinks): + return { + "repo": asdict(o.repo), + "needs": o.needs, # Let the encoder handle the list + } + if isinstance(o, SourceCodeLinks): + return { + "need": o.need, + "links": o.links, + } + if isinstance(o, NeedSourceLinks): + return { + "CodeLinks": o.CodeLinks, + "TestLinks": o.TestLinks, + } + return super().default(o) + + +def RepoSourceLinks_JSON_Decoder( + d: dict[str, Any], +) -> RepoSourceLinks | dict[str, Any]: + if "repo" in d and "needs" in d: + repo = d["repo"] + needs = d["needs"] + return RepoSourceLinks( + repo=RepoInfo( + name=repo.get("name"), + hash=repo.get("hash"), + url=repo.get("url"), + ), + # We know this can only be list[SourceCodeLinks] and nothing else + # Therefore => we ignore the type error here + needs=[SourceCodeLinks_JSON_Decoder(need) for need in needs], # type: ignore + ) + return d + + +def store_repo_source_links_json( + file: Path, source_code_links: list[RepoSourceLinks] +): + # After `rm -rf _build` or on clean builds the directory does not exist, + # so we need to create it. We create any folder that might be missing + file.parent.mkdir(exist_ok=True, parents=True) + with open(file, "w") as f: + json.dump( + source_code_links, + f, + cls=RepoSourceLinks_TEST_JSON_Encoder, + indent=2, + ensure_ascii=False, + ) + + +def load_repo_source_links_json(file: Path) -> list[RepoSourceLinks]: + links: list[RepoSourceLinks] = json.loads( + file.read_text(encoding="utf-8"), + object_hook=RepoSourceLinks_JSON_Decoder, + ) + assert isinstance(links, list), ( + "The RepoSourceLink json should be a list of RepoSourceLink objects." + ) + assert all(isinstance(link, RepoSourceLinks) for link in links), ( + "All items in repo source link cache should be RepoSourceLink objects." + ) + return links + + +def group_needs_by_repo(links: list[SourceCodeLinks]) -> list[RepoSourceLinks]: + repo_groups: dict[str, RepoSourceLinks] = {} + + for source_link in links: + # Check if we can take repoInfo from code or testlinks + if source_link.links.CodeLinks: + first_link = source_link.links.CodeLinks[0] + elif source_link.links.TestLinks: + first_link = source_link.links.TestLinks[0] + else: + # This should not happen? + continue + repo_key = first_link.repo_name + + if repo_key not in repo_groups: + repo_groups[repo_key] = RepoSourceLinks( + repo=RepoInfo( + name=repo_key, hash=first_link.hash, url=first_link.url + ) + ) + + # TODO: Add an assert that checks if needs only are + # in a singular repo (not allowed to be in multiple) + repo_groups[repo_key].needs.append(source_link) + + return [ + RepoSourceLinks(repo=group.repo, needs=group.needs) + for group in repo_groups.values() + ] diff --git a/src/extensions/score_source_code_linker/testlink.py b/src/extensions/score_source_code_linker/testlink.py index ee83c7f95..32f5cca8a 100644 --- a/src/extensions/score_source_code_linker/testlink.py +++ b/src/extensions/score_source_code_linker/testlink.py @@ -33,7 +33,7 @@ LOGGER = logging.get_logger(__name__) -@dataclass(frozen=True, order=True) +@dataclass(order=True) class DataForTestLink: name: str file: Path @@ -42,6 +42,56 @@ class DataForTestLink: verify_type: str result: str result_text: str = "" + repo_name: str = "local_repo" + hash: str = "" + url: str = "" + + # Adding hashing & equality as this is needed to make comparisions. + # Since the Dataclass is not 'frozen = true' it isn't automatically hashable + def __hash__(self): + return hash( + ( + self.name, + str(self.file), + self.line, + self.need, + self.verify_type, + self.result, + self.result_text, + self.repo_name, + self.hash, + self.url, + ) + ) + + def __eq__(self, other: Any): + if not isinstance(other, DataForTestLink): + return NotImplemented + return ( + self.name == other.name + and self.file == other.file + and self.line == other.line + and self.need == other.need + and self.verify_type == other.verify_type + and self.result == other.result + and self.result_text == other.result_text + and self.repo_name == other.repo_name + and self.hash == other.hash + and self.url == other.url + ) + + # Normal 'dictionary conversion'. Converts all fields + def to_dict_full(self) -> dict[str, str | Path | int]: + return asdict(self) + + # Drops MetaData fields for saving the Dataclass (saving space in json) + # The information is in the 'Repo_Source_Link' in the end + def to_dict_without_metadata(self) -> dict[str, str | Path | int]: + d = asdict(self) + d.pop("repo_name", None) + d.pop("hash", None) + d.pop("url", None) + return d class DataForTestLink_JSON_Encoder(json.JSONEncoder): @@ -60,6 +110,9 @@ def DataForTestLink_JSON_Decoder(d: dict[str, Any]) -> DataForTestLink | dict[st "line", "need", "verify_type", + "repo_name", + "hash", + "url", "result", "result_text", } <= d.keys(): @@ -68,6 +121,9 @@ def DataForTestLink_JSON_Decoder(d: dict[str, Any]) -> DataForTestLink | dict[st file=Path(d["file"]), line=d["line"], need=d["need"], + repo_name=d.get("repo_name", ""), + hash=d.get("hash", ""), + url=d.get("url", ""), verify_type=d["verify_type"], result=d["result"], result_text=d["result_text"], @@ -83,6 +139,9 @@ class DataOfTestCase: file: str | None = None line: str | None = None result: str | None = None # passed | falied | skipped | disabled + repo_name: str | None = None + hash: str | None = None + url: str | None = None # Intentionally not snakecase to make dict parsing simple TestType: str | None = None DerivationTechnique: str | None = None @@ -98,6 +157,9 @@ def from_dict(cls, data: dict[str, Any]): # type-ignore file=data.get("file"), line=data.get("line"), result=data.get("result"), + repo_name=data.get("repo_name"), + hash=data.get("hash"), + url=data.get("url"), TestType=data.get("TestType"), DerivationTechnique=data.get("DerivationTechnique"), result_text=data.get("result_text"), @@ -158,6 +220,8 @@ def is_valid(self) -> bool: # and self.TestType is not None # and self.DerivationTechnique is not None # ): + # Hash & URL are explictily allowed to be empty but not none. + # repo_name has to be always filled or something went wrong fields = [ x for x in self.__dataclass_fields__ @@ -199,6 +263,9 @@ def parse_attributes(verify_field: str | None, verify_type: str): assert self.file is not None assert self.line is not None assert self.result is not None + assert self.repo_name is not None + assert self.hash is not None + assert self.url is not None assert self.result_text is not None assert self.TestType is not None assert self.DerivationTechnique is not None @@ -212,6 +279,9 @@ def parse_attributes(verify_field: str | None, verify_type: str): verify_type=verify_type, result=self.result, result_text=self.result_text, + repo_name=self.repo_name, + hash=self.hash, + url=self.url, ) return list( diff --git a/src/extensions/score_source_code_linker/tests/expected_codelink.json b/src/extensions/score_source_code_linker/tests/expected_codelink.json index 447ef8a69..422e88352 100644 --- a/src/extensions/score_source_code_linker/tests/expected_codelink.json +++ b/src/extensions/score_source_code_linker/tests/expected_codelink.json @@ -4,27 +4,39 @@ "line": 3, "tag":"#-----req-Id:", "need": "TREQ_ID_1", - "full_line": "#-----req-Id: TREQ_ID_1" + "full_line": "#-----req-Id: TREQ_ID_1", + "repo_name": "local_repo", + "hash": "", + "url": "" }, { "file": "src/implementation2.py", "line": 5, "tag":"#-----req-Id:", "need": "TREQ_ID_1", - "full_line": "#-----req-Id: TREQ_ID_1" + "full_line": "#-----req-Id: TREQ_ID_1", + "repo_name": "local_repo", + "hash": "", + "url": "" }, { "file": "src/implementation1.py", "line": 9, "tag":"#-----req-Id:", "need": "TREQ_ID_2", - "full_line":"#-----req-Id: TREQ_ID_2" + "full_line":"#-----req-Id: TREQ_ID_2", + "repo_name": "local_repo", + "hash": "", + "url": "" }, { "file": "src/bad_implementation.py", "line":2, "tag":"#-----req-Id:", "need": "TREQ_ID_200", - "full_line":"#-----req-Id: TREQ_ID_200" + "full_line":"#-----req-Id: TREQ_ID_200", + "repo_name": "local_repo", + "hash": "", + "url": "" } ] diff --git a/src/extensions/score_source_code_linker/tests/expected_grouped.json b/src/extensions/score_source_code_linker/tests/expected_grouped.json index 256232661..e2668e476 100644 --- a/src/extensions/score_source_code_linker/tests/expected_grouped.json +++ b/src/extensions/score_source_code_linker/tests/expected_grouped.json @@ -8,14 +8,20 @@ "line": 3, "tag":"#-----req-Id:", "need": "TREQ_ID_1", - "full_line": "#-----req-Id: TREQ_ID_1" + "full_line": "#-----req-Id: TREQ_ID_1", + "repo_name": "local_repo", + "hash": "", + "url": "" }, { "file": "src/implementation2.py", "line": 5, "tag":"#-----req-Id:", "need": "TREQ_ID_1", - "full_line": "#-----req-Id: TREQ_ID_1" + "full_line": "#-----req-Id: TREQ_ID_1", + "repo_name": "local_repo", + "hash": "", + "url": "" } ], @@ -27,7 +33,10 @@ "need": "TREQ_ID_1", "verify_type": "fully", "result": "passed", - "result_text": "" + "result_text": "", + "repo_name": "local_repo", + "hash": "", + "url": "" } ] } @@ -41,7 +50,10 @@ "line": 9, "tag":"#-----req-Id:", "need": "TREQ_ID_2", - "full_line":"#-----req-Id: TREQ_ID_2" + "full_line":"#-----req-Id: TREQ_ID_2", + "repo_name": "local_repo", + "hash": "", + "url": "" } ], "TestLinks": [ @@ -53,7 +65,10 @@ "need": "TREQ_ID_2", "verify_type": "partially", "result": "passed", - "result_text": "" + "result_text": "", + "repo_name": "local_repo", + "hash": "", + "url": "" }, { "name": "test_error_handling", @@ -62,7 +77,10 @@ "need": "TREQ_ID_2", "verify_type": "partially", "result": "passed", - "result_text": "" + "result_text": "", + "repo_name": "local_repo", + "hash": "", + "url": "" } ] @@ -80,7 +98,10 @@ "need": "TREQ_ID_3", "verify_type": "partially", "result": "passed", - "result_text": "" + "result_text": "", + "repo_name": "local_repo", + "hash": "", + "url": "" }, { "name": "test_error_handling", @@ -89,7 +110,10 @@ "need": "TREQ_ID_3", "verify_type": "partially", "result": "passed", - "result_text": "" + "result_text": "", + "repo_name": "local_repo", + "hash": "", + "url": "" } ] } @@ -103,7 +127,10 @@ "line":2, "tag":"#-----req-Id:", "need": "TREQ_ID_200", - "full_line":"#-----req-Id: TREQ_ID_200" + "full_line":"#-----req-Id: TREQ_ID_200", + "repo_name": "local_repo", + "hash": "", + "url": "" } ], diff --git a/src/extensions/score_source_code_linker/tests/expected_repo_grouped.json b/src/extensions/score_source_code_linker/tests/expected_repo_grouped.json new file mode 100644 index 000000000..3d0523121 --- /dev/null +++ b/src/extensions/score_source_code_linker/tests/expected_repo_grouped.json @@ -0,0 +1,83 @@ +[ + { + "repo": { + "name": "local_repo", + "hash": "", + "url": "" + }, + "needs": [ + { + "need": "MOD_REQ_1", + "links": { + "CodeLinks": [ + { + "file": "src/repo_a_impl.py", + "line": 3, + "tag": "#-----req-Id:", + "need": "MOD_REQ_1", + "full_line": "#-----req-Id: MOD_REQ_1" + }, + { + "file": "src/repo_b_impl.py", + "line": 3, + "tag": "#-----req-Id:", + "need": "MOD_REQ_1", + "full_line": "#-----req-Id: MOD_REQ_1" + } + ], + "TestLinks": [ + { + "name": "test_repo_a", + "file": "src/test_repo_a.py", + "line": 10, + "need": "MOD_REQ_1", + "verify_type": "fully", + "result": "passed", + "result_text": "" + } + ] + } + }, + { + "need": "MOD_REQ_2", + "links": { + "CodeLinks": [ + { + "file": "src/repo_a_impl.py", + "line": 7, + "tag": "#-----req-Id:", + "need": "MOD_REQ_2", + "full_line": "#-----req-Id: MOD_REQ_2" + } + ], + "TestLinks": [] + } + }, + { + "need": "MOD_REQ_3", + "links": { + "CodeLinks": [ + { + "file": "src/repo_b_impl.py", + "line": 7, + "tag": "#-----req-Id:", + "need": "MOD_REQ_3", + "full_line": "#-----req-Id: MOD_REQ_3" + } + ], + "TestLinks": [ + { + "name": "test_repo_b", + "file": "src/test_repo_b.py", + "line": 20, + "need": "MOD_REQ_3", + "verify_type": "partially", + "result": "passed", + "result_text": "" + } + ] + } + } + ] + } +] diff --git a/src/extensions/score_source_code_linker/tests/expected_testlink.json b/src/extensions/score_source_code_linker/tests/expected_testlink.json index 19068a4d5..23fbbe395 100644 --- a/src/extensions/score_source_code_linker/tests/expected_testlink.json +++ b/src/extensions/score_source_code_linker/tests/expected_testlink.json @@ -6,7 +6,10 @@ "need": "TREQ_ID_2", "verify_type": "partially", "result": "passed", - "result_text": "" + "result_text": "", + "repo_name": "local_repo", + "hash": "", + "url": "" }, { "name": "test_api_response_format", @@ -15,7 +18,10 @@ "need": "TREQ_ID_3", "verify_type": "partially", "result": "passed", - "result_text": "" + "result_text": "", + "repo_name": "local_repo", + "hash": "", + "url": "" }, { "name": "test_error_handling", @@ -24,7 +30,10 @@ "need": "TREQ_ID_2", "verify_type": "partially", "result": "passed", - "result_text": "" + "result_text": "", + "repo_name": "local_repo", + "hash": "", + "url": "" }, { "name": "test_error_handling", @@ -33,7 +42,10 @@ "need": "TREQ_ID_3", "verify_type": "partially", "result": "passed", - "result_text": "" + "result_text": "", + "repo_name": "local_repo", + "hash": "", + "url": "" }, { "name": "TestRequirementsCoverage__test_system_startup_time", @@ -42,6 +54,9 @@ "need": "TREQ_ID_1", "verify_type": "fully", "result": "passed", - "result_text": "" + "result_text": "", + "repo_name": "local_repo", + "hash": "", + "url": "" } ] diff --git a/src/extensions/score_source_code_linker/tests/test_codelink.py b/src/extensions/score_source_code_linker/tests/test_codelink.py index 29ddc7235..b6a277335 100644 --- a/src/extensions/score_source_code_linker/tests/test_codelink.py +++ b/src/extensions/score_source_code_linker/tests/test_codelink.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2025 Contributors to the Eclipse Foundation +# Copyright (c) 2026 Contributors to the Eclipse Foundation # # See the NOTICE file(s) distributed with this work for additional # information regarding copyright ownership. @@ -10,6 +10,11 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* + +# โ•“ โ•– +# โ•‘ Some portions created by Co-Pilot โ•‘ +# โ•™ โ•œ + import json import os import subprocess @@ -20,7 +25,9 @@ from typing import Any import pytest -from attribute_plugin import add_test_properties # type: ignore[import-untyped] + +# S-CORE plugin to allow for properties/attributes in xml +# Enables Testlinking from sphinx_needs.data import NeedsInfoType, NeedsMutable from sphinx_needs.need_item import ( NeedItem, @@ -28,35 +35,29 @@ NeedsContent, ) -# Import the module under test -# Note: You'll need to adjust these imports based on your actual module structure from src.extensions.score_source_code_linker import ( find_need, get_cache_filename, group_by_need, ) +from src.extensions.score_source_code_linker.helpers import ( + get_github_link, +) from src.extensions.score_source_code_linker.needlinks import ( + MetaData, NeedLink, + NeedLinkEncoder, + is_metadata, load_source_code_links_json, + load_source_code_links_with_metadata_json, + needlink_decoder, store_source_code_links_json, + store_source_code_links_with_metadata_json, ) +from src.extensions.score_source_code_linker.repo_source_links import RepoInfo from src.helper_lib import ( get_current_git_hash, ) -from src.helper_lib.additional_functions import get_github_link - -""" -# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ATTENTIONโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - -# โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ -# โ”‚ !!!!! โ”‚ -# โ”‚ BOILERPLATE TEST MADE VIA โ”‚ -# โ”‚ GENERATION. NOT YET FULLY LOOKED โ”‚ -# โ”‚ THROUGH โ”‚ -# โ”‚ !!!! โ”‚ -# โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ - -""" def test_need(**kwargs: Any) -> NeedItem: @@ -127,13 +128,28 @@ def default(self, o: object): def needlink_test_decoder(d: dict[str, Any]) -> NeedLink | dict[str, Any]: - if {"file", "line", "tag", "need", "full_line"} <= d.keys(): + """ + Since we have our own decoder, we have to ensure it works as expected + """ + if { + "file", + "line", + "tag", + "need", + "full_line", + "repo_name", + "hash", + "url", + } <= d.keys(): return NeedLink( file=Path(d["file"]), line=d["line"], tag=decode_comment(d["tag"]), need=d["need"], full_line=decode_comment(d["full_line"]), + repo_name=d.get("repo_name", ""), + hash=d.get("hash", ""), + url=d.get("url", ""), ) # It's something else, pass it on to other decoders return d @@ -184,6 +200,9 @@ def sample_needlinks() -> list[NeedLink]: tag="#" + " req-Id:", need="TREQ_ID_1", full_line="#" + " req-Id: TREQ_ID_1", + repo_name="local_repo", + hash="", + url="", ), NeedLink( file=Path("src/implementation2.py"), @@ -191,6 +210,9 @@ def sample_needlinks() -> list[NeedLink]: tag="#" + " req-Id:", need="TREQ_ID_1", full_line="#" + " req-Id: TREQ_ID_1", + repo_name="local_repo", + hash="", + url="", ), NeedLink( file=Path("src/implementation1.py"), @@ -198,6 +220,9 @@ def sample_needlinks() -> list[NeedLink]: tag="#" + " req-Id:", need="TREQ_ID_2", full_line="#" + " req-Id: TREQ_ID_2", + repo_name="local_repo", + hash="", + url="", ), NeedLink( file=Path("src/bad_implementation.py"), @@ -205,6 +230,9 @@ def sample_needlinks() -> list[NeedLink]: tag="#" + " req-Id:", need="TREQ_ID_200", full_line="#" + " req-Id: TREQ_ID_200", + repo_name="local_repo", + hash="", + url="", ), ] @@ -239,12 +267,6 @@ def sample_needs() -> dict[str, dict[str, str]]: } -# Test utility functions -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_get_cache_filename(): """Test cache filename generation.""" build_dir = Path("/tmp/build") @@ -253,17 +275,13 @@ def test_get_cache_filename(): assert result == expected +# Done to appease the LSP Gods def make_needs(needs_dict: dict[str, dict[str, Any]]) -> NeedsMutable: return NeedsMutable( {need_id: test_need(**params) for need_id, params in needs_dict.items()} ) -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_find_need_direct_match(): """Test finding a need with direct ID match.""" all_needs = make_needs( @@ -277,11 +295,6 @@ def test_find_need_direct_match(): assert result["id"] == "REQ_001" -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_find_need_not_found(): """Test finding a need that doesn't exist.""" all_needs = make_needs( @@ -294,11 +307,6 @@ def test_find_need_not_found(): assert result is None -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_group_by_need(sample_needlinks: list[NeedLink]) -> None: """Test grouping source code links by need ID.""" result = group_by_need(sample_needlinks) @@ -318,54 +326,12 @@ def test_group_by_need(sample_needlinks: list[NeedLink]) -> None: assert len(found_link.links.CodeLinks) == 1 -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_group_by_need_empty_list(): """Test grouping empty list of needlinks.""" result = group_by_need([], []) assert len(result) == 0 -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) -def test_get_github_link_with_real_repo(git_repo: Path) -> None: - """Test generating GitHub link with real repository.""" - # Create a needlink - needlink = NeedLink( - file=Path("src/test.py"), - line=42, - tag="#" + " req-Id:", - need="REQ_001", - full_line="#" + " req-Id: REQ_001", - ) - - # Have to change directories in order to ensure that we get the right/any .git file - os.chdir(Path(git_repo).absolute()) - result = get_github_link(needlink) - - # Should contain the base URL, hash, file path, and line number - assert "https://github.com/test-user/test-repo/blob/" in result - assert "src/test.py#L42" in result - assert len(result.split("/")) >= 7 # Should have proper URL structure - - # Verify the hash is actually from the repo - hash_from_link = result.split("/blob/")[1].split("/")[0] - actual_hash = get_current_git_hash(git_repo) - assert hash_from_link == actual_hash - - -# Test cache file operations -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_cache_file_operations( temp_dir: Path, sample_needlinks: list[NeedLink] ) -> None: @@ -392,11 +358,6 @@ def test_cache_file_operations( assert loaded_links[3].line == 2 -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_cache_file_with_encoded_comments(temp_dir: Path) -> None: """Test that cache file properly handles encoded comments.""" # Create needlinks with spaces in tags and full_line @@ -407,6 +368,9 @@ def test_cache_file_with_encoded_comments(temp_dir: Path) -> None: tag="#" + " req-Id:", need="TEST_001", full_line="#" + " req-Id: TEST_001", + repo_name="local_repo", + hash="", + url="", ) ] @@ -426,14 +390,6 @@ def test_cache_file_with_encoded_comments(temp_dir: Path) -> None: assert loaded_links[0].full_line == "#" + " req-Id: TEST_001" -# Integration tests - - -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_group_by_need_and_find_need_integration( sample_needlinks: list[NeedLink], ) -> None: @@ -457,11 +413,6 @@ def test_group_by_need_and_find_need_integration( assert found_need["id"] == found_link.need -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_source_linker_end_to_end_with_real_files( temp_dir: Path, git_repo: Path ) -> None: @@ -470,7 +421,7 @@ def test_source_linker_end_to_end_with_real_files( src_dir: Path = git_repo / "src" src_dir.mkdir() - (src_dir / "implementation1.py").write_text( + _ = (src_dir / "implementation1.py").write_text( """ # Some implementation #""" @@ -486,7 +437,7 @@ def function2(): """ ) - (src_dir / "implementation2.py").write_text( + _ = (src_dir / "implementation2.py").write_text( """ # Another implementation #""" @@ -497,8 +448,8 @@ def another_function(): ) # Commit the changes - subprocess.run(["git", "add", "."], cwd=git_repo, check=True) - subprocess.run( + _ = subprocess.run(["git", "add", "."], cwd=git_repo, check=True) + _ = subprocess.run( ["git", "commit", "-m", "Add implementation files"], cwd=git_repo, check=True ) @@ -511,6 +462,9 @@ def another_function(): tag="#" + " req-Id:", need="TREQ_ID_1", full_line="#" + " req-Id: TREQ_ID_1", + repo_name="local_repo", + hash="", + url="", ), NeedLink( file=Path("src/implementation1.py"), @@ -518,6 +472,9 @@ def another_function(): tag="#" + " req-Id:", need="TREQ_ID_2", full_line="#" + " req-Id: TREQ_ID_2", + repo_name="local_repo", + hash="", + url="", ), NeedLink( file=Path("src/implementation2.py"), @@ -525,6 +482,9 @@ def another_function(): tag="#" + " req-Id:", need="TREQ_ID_1", full_line="#" + " req-Id: TREQ_ID_1", + repo_name="local_repo", + hash="", + url="", ), ] @@ -548,17 +508,13 @@ def another_function(): # Test GitHub link generation # Have to change directories in order to ensure that we get the right/any .git file os.chdir(Path(git_repo).absolute()) + metadata = RepoInfo(name="local_repo", hash="", url="") for needlink in loaded_links: - github_link = get_github_link(needlink) + github_link = get_github_link(metadata, needlink) assert "https://github.com/test-user/test-repo/blob/" in github_link assert f"src/{needlink.file.name}#L{needlink.line}" in github_link -@add_test_properties( - partially_verifies=["tool_req__docs_dd_link_source_code_link"], - test_type="requirements-based", - derivation_technique="requirements-analysis", -) def test_multiple_commits_hash_consistency(git_repo: Path) -> None: """Test that git hash remains consistent and links update properly.""" # Get initial hash @@ -566,9 +522,11 @@ def test_multiple_commits_hash_consistency(git_repo: Path) -> None: # Create and commit a new file new_file: Path = git_repo / "new_file.py" - new_file.write_text("# New file\nprint('new')") - subprocess.run(["git", "add", "."], cwd=git_repo, check=True) - subprocess.run(["git", "commit", "-m", "Add new file"], cwd=git_repo, check=True) + _ = new_file.write_text("# New file\nprint('new')") + _ = subprocess.run(["git", "add", "."], cwd=git_repo, check=True) + _ = subprocess.run( + ["git", "commit", "-m", "Add new file"], cwd=git_repo, check=True + ) # Get new hash new_hash = get_current_git_hash(git_repo) @@ -586,6 +544,447 @@ def test_multiple_commits_hash_consistency(git_repo: Path) -> None: full_line="#" + " req-Id: TREQ_ID_1", ) + metadata = RepoInfo(name="local_repo", hash="", url="") os.chdir(Path(git_repo).absolute()) - github_link = get_github_link(needlink) + github_link = get_github_link(metadata, needlink) assert new_hash in github_link + + +def test_is_metadata_missing_keys(): + """Bad path: Dict missing required keys returns False""" + incomplete = {"repo_name": "test", "hash": "abc"} + assert is_metadata(incomplete) is False + + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€[ NeedLink Dataclass Tests ]โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +def test_needlink_to_dict_without_metadata(): + """to_dict_without_metadata should return NeedLink without metadata""" + needlink = NeedLink( + file=Path("src/test.py"), + line=10, + tag="#" + " req-Id:", + need="REQ_1", + full_line="#" + " req-Id: REQ_1", + repo_name="test_repo", + url="https://github.com/test/repo", + hash="abc123", + ) + result = needlink.to_dict_without_metadata() + + assert "repo_name" not in result + assert "hash" not in result + assert "url" not in result + assert result["need"] == "REQ_1" + assert result["line"] == 10 + + +def test_needlink_to_dict_full(): + """to_dict_full includes all fields""" + needlink = NeedLink( + file=Path("src/test.py"), + line=10, + tag="#" + " req-Id:", + need="REQ_1", + full_line="#" + " req-Id: REQ_1", + repo_name="test_repo", + url="https://github.com/test/repo", + hash="abc123", + ) + result = needlink.to_dict_full() + + assert result["repo_name"] == "test_repo" + assert result["hash"] == "abc123" + assert result["url"] == "https://github.com/test/repo" + assert result["need"] == "REQ_1" + + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€[ JSON Encoder Tests ]โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +def test_needlink_encoder_includes_metadata(): + """Encoder includes all fields""" + encoder = NeedLinkEncoder() + needlink = NeedLink( + file=Path("src/test.py"), + line=10, + tag="#" + " req-Id:", + need="REQ_1", + full_line="#" + " req-Id: REQ_1", + repo_name="test_repo", + url="https://github.com/test/repo", + hash="abc123", + ) + result = encoder.default(needlink) + + assert isinstance(result, dict) + assert result["repo_name"] == "test_repo" + assert result["hash"] == "abc123" + assert result["url"] == "https://github.com/test/repo" + + +# ============================================================================ +# JSON Decoder Tests +# ============================================================================ + + +def test_needlink_decoder_with_all_fields(): + """Decoder reconstructs NeedLink with all fields""" + json_data: dict[str, Any] = { + "file": "src/test.py", + "line": 10, + "tag": "#" + " req-Id:", + "need": "REQ_1", + "full_line": "#" + " req-Id: REQ_1", + "repo_name": "test_repo", + "hash": "abc123", + "url": "https://github.com/test/repo", + } + result = needlink_decoder(json_data) + + assert isinstance(result, NeedLink) + assert result.file == Path("src/test.py") + assert result.line == 10 + assert result.repo_name == "test_repo" + + +def test_needlink_decoder_non_needlink_dict(): + """Edge case: Non-NeedLink dict is returned unchanged""" + json_data = {"some": "other", "data": "structure"} + result = needlink_decoder(json_data) + assert result == json_data + + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€[ Testing Encoding / Decoding ]โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +def test_load_validates_list_type(tmp_path: Path): + """Bad path: Non-list JSON fails validation""" + test_file = tmp_path / "not_list.json" + _ = test_file.write_text('{"file": "src/test.py", "line": 1}') + + with pytest.raises(AssertionError, match="should be a list"): + load_source_code_links_json(test_file) + + +def test_load_validates_all_items_are_needlinks(tmp_path: Path): + """Bad path: List with invalid items fails validation""" + test_file = tmp_path / "invalid_items.json" + _ = test_file.write_text('[{"invalid": "item"}]') + + with pytest.raises(AssertionError, match="should be NeedLink objects"): + load_source_code_links_json(test_file) + + +# Adding additional tests to test the metadata stuff (excluding it if wanted) + + +def test_store_and_load_with_metadata(tmp_path: Path): + """Happy path: Store with metadata dict and load correctly""" + metadata: MetaData = { + "repo_name": "external_repo", + "hash": "commit_xyz", + "url": "https://github.com/external/repo", + } + needlinks = [ + NeedLink( + file=Path("src/impl.py"), + line=10, + tag="#" + " req-Id:", + need="REQ_1", + full_line="#" + " req-Id: REQ_1", + repo_name="", # Will be filled from metadata + url="", + hash="", + ), + NeedLink( + file=Path("src/impl2.py"), + line=20, + tag="#" + " req-Id:", + need="REQ_2", + full_line="#" + " req-Id: REQ_2", + repo_name="", + url="", + hash="", + ), + ] + + test_file = tmp_path / "with_metadata.json" + store_source_code_links_with_metadata_json(test_file, metadata, needlinks) + loaded = load_source_code_links_with_metadata_json(test_file) + + assert len(loaded) == 2 + # Verify metadata was applied to all links + for nl in loaded: + assert nl.repo_name == "external_repo" + assert nl.url == "https://github.com/external/repo" + assert nl.hash == "commit_xyz" + + +def test_load_with_metadata_missing_metadata_dict(tmp_path: Path): + """ + Test if loading file without metadata dict via + the scl_with_metadata loader raises TypeError + """ + # Store without metadata (just needlinks) + needlinks = [ + NeedLink( + file=Path("src/test.py"), + line=1, + tag="#" + " req-Id:", + need="REQ_1", + full_line="#" + " req-Id: REQ_1", + ) + ] + + test_file = tmp_path / "no_metadata.json" + store_source_code_links_json(test_file, needlinks) + + # Try to load as if it has metadata + with pytest.raises(TypeError, match="you might wanted to call"): + load_source_code_links_with_metadata_json(test_file) + + +def test_load_with_metadata_invalid_items_after_metadata(tmp_path: Path): + """Test wrong JSON structure""" + test_file = tmp_path / "bad_items.json" + # Manually create invalid JSON + _ = test_file.write_text( + json.dumps( + [{"repo_name": "mod", "hash": "h", "url": "u"}, {"invalid": "structure"}] + ) + ) + + with pytest.raises(TypeError, match="must decode to NeedLink objects"): + load_source_code_links_with_metadata_json(test_file) + + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€[ File Path Resolution Tests ]โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +def test_load_resolves_relative_path_with_env_var( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + """Test if relative path is resolved using BUILD_WORKSPACE_DIRECTORY""" + workspace = tmp_path / "workspace" + workspace.mkdir() + + needlinks = [ + NeedLink( + file=Path("src/test.py"), + line=1, + tag="#" + " req-Id:", + need="REQ_1", + full_line="#" + " req-Id: REQ_1", + ) + ] + + # Store in workspace + cache_file = workspace / "cache.json" + store_source_code_links_json(cache_file, needlinks) + + # Set env var and load with relative path + monkeypatch.setenv("BUILD_WORKSPACE_DIRECTORY", str(workspace)) + loaded = load_source_code_links_json(Path("cache.json")) + + assert len(loaded) == 1 + assert loaded[0].need == "REQ_1" + + +def test_load_with_metadata_resolves_relative_path( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + """Edge case: load_with_metadata resolves relative paths using env var""" + workspace = tmp_path / "workspace" + workspace.mkdir() + + metadata: MetaData = { + "repo_name": "mod", + "hash": "h", + "url": "u", + } + needlinks = [ + NeedLink( + file=Path("src/test.py"), + line=1, + tag="#" + " req-Id:", + need="REQ_1", + full_line="#" + " req-Id: REQ_1", + ) + ] + + cache_file = workspace / "metadata_cache.json" + store_source_code_links_with_metadata_json(cache_file, metadata, needlinks) + + monkeypatch.setenv("BUILD_WORKSPACE_DIRECTORY", str(workspace)) + loaded = load_source_code_links_with_metadata_json(Path("metadata_cache.json")) + + assert len(loaded) == 1 + assert loaded[0].repo_name == "mod" + + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€[ Roundtrip Tests ]โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +def test_roundtrip_standard_format(tmp_path: Path): + """Happy path: Standard format preserves all data""" + needlinks = [ + NeedLink( + file=Path("src/file1.py"), + line=10, + tag="#" + " req-Id:", + need="REQ_A", + full_line="#" + " req-Id: REQ_A", + repo_name="mod_a", + url="url_a", + hash="hash_a", + ), + NeedLink( + file=Path("src/file2.py"), + line=20, + tag="#" + " req-Id:", + need="REQ_B", + full_line="#" + " req-Id: REQ_B", + repo_name="mod_b", + url="url_b", + hash="hash_b", + ), + ] + + test_file = tmp_path / "standard.json" + store_source_code_links_json(test_file, needlinks) + loaded = load_source_code_links_json(test_file) + + assert len(loaded) == 2 + assert needlinks == loaded + assert loaded[0].repo_name == "mod_a" + assert loaded[1].repo_name == "mod_b" + + +def test_roundtrip_metadata_format_applies_metadata(tmp_path: Path): + """Happy path: Metadata format applies metadata to all links""" + metadata: MetaData = { + "repo_name": "shared_repo", + "hash": "shared_hash", + "url": "https://github.com/shared/repo", + } + needlinks = [ + NeedLink( + file=Path("src/f1.py"), + line=5, + tag="#" + " req-Id:", + need="R1", + full_line="#" + " req-Id: R1", + ), + NeedLink( + file=Path("src/f2.py"), + line=15, + tag="#" + " req-Id:", + need="R2", + full_line="#" + " req-Id: R2", + ), + ] + + test_file = tmp_path / "with_metadata.json" + store_source_code_links_with_metadata_json(test_file, metadata, needlinks) + loaded = load_source_code_links_with_metadata_json(test_file) + + assert len(loaded) == 2 + # Both should have the same metadata applied + for link in loaded: + assert link.repo_name == "shared_repo" + assert link.hash == "shared_hash" + assert link.url == "https://github.com/shared/repo" + + +def test_roundtrip_empty_lists(tmp_path: Path): + """Edge case: Empty list can be stored and loaded""" + test_file = tmp_path / "empty.json" + store_source_code_links_json(test_file, []) + loaded = load_source_code_links_json(test_file) + + assert loaded == [] + + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€[ JSON Format Verification Tests ]โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +def test_json_format_with_metadata_has_separate_dict(tmp_path: Path): + """Edge case: Verify metadata format has metadata as first element""" + metadata: MetaData = { + "repo_name": "test_mod", + "hash": "test_hash", + "url": "test_url", + } + needlink = NeedLink( + file=Path("src/test.py"), + line=1, + tag="#" + " req-Id:", + need="REQ_1", + full_line="#" + " req-Id: REQ_1", + ) + + test_file = tmp_path / "metadata_format.json" + store_source_code_links_with_metadata_json(test_file, metadata, [needlink]) + + with open(test_file) as f: + raw_json = json.load(f) + + assert isinstance(raw_json, list) + assert len(raw_json) == 2 # metadata + 1 needlink + # First element should be metadata dict + assert raw_json[0]["repo_name"] == "test_mod" + assert raw_json[0]["hash"] == "test_hash" + assert raw_json[0]["url"] == "test_url" + + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€[ NeedLink Equality and Hashing Tests ]โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +def test_needlink_equality_same_values(): + """Happy path: Two NeedLinks with same values are equal""" + link1 = NeedLink( + file=Path("src/test.py"), + line=10, + tag="#" + " req-Id:", + need="REQ_1", + full_line="#" + " req-Id: REQ_1", + repo_name="mod", + url="url", + hash="hash", + ) + link2 = NeedLink( + file=Path("src/test.py"), + line=10, + tag="#" + " req-Id:", + need="REQ_1", + full_line="#" + " req-Id: REQ_1", + repo_name="mod", + url="url", + hash="hash", + ) + + assert link1 == link2 + assert hash(link1) == hash(link2) + + +def test_needlink_inequality_different_values(): + """Edge case: NeedLinks with different values are not equal""" + link1 = NeedLink( + file=Path("src/test.py"), + line=10, + tag="#" + " req-Id:", + need="REQ_1", + full_line="#" + " req-Id: REQ_1", + ) + link2 = NeedLink( + file=Path("src/test.py"), + line=20, # Different line + tag="#" + " req-Id:", + need="REQ_1", + full_line="#" + " req-Id: REQ_1", + ) + + assert link1 != link2 + assert hash(link1) != hash(link2) diff --git a/src/extensions/score_source_code_linker/tests/test_helpers.py b/src/extensions/score_source_code_linker/tests/test_helpers.py new file mode 100644 index 000000000..ab5207bb3 --- /dev/null +++ b/src/extensions/score_source_code_linker/tests/test_helpers.py @@ -0,0 +1,351 @@ +# ******************************************************************************* +# Copyright (c) 2025 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import json +import os +import subprocess +import tempfile +from collections.abc import Generator +from pathlib import Path + +import pytest + +from src.extensions.score_source_code_linker.helpers import ( + get_github_link, + get_github_link_from_json, + parse_info_from_known_good, + parse_repo_name_from_path, +) +from src.extensions.score_source_code_linker.needlinks import DefaultNeedLink +from src.extensions.score_source_code_linker.repo_source_links import RepoInfo +from src.helper_lib import get_current_git_hash + +# โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ +# โ”‚ Tests for parse_repo_name_from_path โ”‚ +# โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ + + +def test_parse_repo_name_from_external_path(): + """Test parsing repo name from external/combo build path.""" + path = Path("external/score_docs_as_code+/src/helper_lib/test_helper_lib.py") + + result = parse_repo_name_from_path(path) + + assert result == "score_docs_as_code" + + +def test_parse_repo_name_from_external_path_2(): + """Test that an extra 'external' in the path does not change the output""" + path = Path( + "external/score_docs_as_code+/src/external/helper_lib/test_helper_lib.py" + ) + + result = parse_repo_name_from_path(path) + + assert result == "score_docs_as_code" + + +def test_parse_repo_name_from_local_path(): + """Test parsing repo name from local build path.""" + path = Path("src/helper_lib/test_helper_lib.py") + + result = parse_repo_name_from_path(path) + + assert result == "local_repo" + + +def test_parse_repo_name_from_empty_path(): + """Test parsing repo name from an empty path.""" + path = Path("") + + result = parse_repo_name_from_path(path) + + assert result == "local_repo" + + +def test_parse_repo_name_without_plus_suffix(): + """ + Test parsing external repo without plus suffix. + This should never happen (due to bazel adding the '+') + But testing this edge case anyway + """ + path = Path("external/repo_without_plus/file.py") + + result = parse_repo_name_from_path(path) + + assert result == "repo_without_plus" + + +# โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ +# โ”‚ PARSE INFO FROM KNOWN GOOD โ”‚ +# โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ + + +# Example of a minimal valid known_good.json structure +VALID_KNOWN_GOOD = { + "modules": { + "target_sw": { + "score_baselibs": { + "repo": "https://github.com/eclipse-score/baselibs.git", + "hash": "158fe6a7b791c58f6eac5f7e4662b8db0cf9ac6e", + }, + "score_communication": { + "repo": "https://github.com/eclipse-score/communication.git", + "hash": "56448a5589a5f7d3921b873e8127b824a8c1ca95", + }, + }, + "tooling": { + "score_docs_as_code": { + "repo": "https://github.com/eclipse-score/docs-as-code.git", + "hash": "c1207676afe6cafd25c35d420e73279a799515d8", + } + }, + } +} + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€[ Test Functions ]โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +@pytest.fixture +def known_good_json(tmp_path: Path): + """Fixture providing a valid known_good.json file.""" + json_file = tmp_path / "known_good.json" + _ = json_file.write_text(json.dumps(VALID_KNOWN_GOOD)) + return json_file + + +# Tests for parse_info_from_known_good +def test_parse_info_from_known_good_happy_path(known_good_json: Path): + """Test parsing repo info from valid known_good.json.""" + hash_result, repo_result = parse_info_from_known_good( + known_good_json, "score_baselibs" + ) + + assert hash_result == "158fe6a7b791c58f6eac5f7e4662b8db0cf9ac6e" + assert repo_result == "https://github.com/eclipse-score/baselibs" + + +def test_parse_info_from_known_good_different_category(known_good_json: Path): + """Test finding repo in different category.""" + hash_result, repo_result = parse_info_from_known_good( + known_good_json, "score_docs_as_code" + ) + + assert hash_result == "c1207676afe6cafd25c35d420e73279a799515d8" + assert repo_result == "https://github.com/eclipse-score/docs-as-code" + + +def test_parse_info_from_known_good_repo_not_found(known_good_json: Path): + """Test that KeyError is raised when repo doesn't exist.""" + with pytest.raises(KeyError, match="Module nonexistent not found"): + parse_info_from_known_good(known_good_json, "nonexistent") + + +def test_parse_info_from_known_good_empty_json(tmp_path: Path): + """Test that it errors when given an empty JSON file.""" + json_file = tmp_path / "example.json" + _ = json_file.write_text("{}") + + with pytest.raises( + AssertionError, + match=f"Known good json at: {json_file} is empty. This is not allowed", + ): + parse_info_from_known_good(json_file, "any_repo") + + +def test_parse_info_from_known_good_no_repo_in_json(tmp_path: Path): + """Test that assertion works when repo not in top level keys""" + json_file = tmp_path / "example.json" + _ = json_file.write_text( + '{"another_key": {"a": "b"}, "second_key": ["a" , "b", "c"]}' + ) + + with pytest.raises( + AssertionError, + match=f"Known good json at: {json_file} is missing the 'modules' key", + ): + parse_info_from_known_good(json_file, "any_repo") + + +def test_parse_info_from_known_good_empty_repo_dict_in_json(tmp_path: Path): + """Test that assertion works if repo dictionary is empty""" + json_file = tmp_path / "emample.json" + _ = json_file.write_text('{"another_key": {"a": "b"}, "modules": {}}') + + with pytest.raises( + AssertionError, + match=f"Known good json at: {json_file} has an empty 'modules' dictionary", + ): + parse_info_from_known_good(json_file, "any_repo") + + +# Tests for get_github_link_from_json +def test_get_github_link_from_json_happy_path(): + """ + Test generating GitHub link with fully filled metadata + =>this happens in combo builds + """ + metadata = RepoInfo( + name="project_name", + url="https://github.com/eclipse/project", + hash="commit123abc", + ) + + link = DefaultNeedLink() + link.file = Path("docs/index.rst") + link.line = 100 + + result = get_github_link_from_json(metadata, link) + + assert ( + result + == "https://github.com/eclipse/project/blob/commit123abc/docs/index.rst#L100" + ) + + +def test_get_github_link_from_json_with_none_link(): + """Test that None link creates DefaultNeedLink.""" + metadata = RepoInfo( + name="test_repo", url="https://github.com/test/repo", hash="def456" + ) + + result = get_github_link_from_json(metadata, None) + + assert result.startswith("https://github.com/test/repo/blob/def456/") + assert "#L" in result + + +def test_get_github_link_from_json_with_line_zero(): + """Test generating link with line number 0.""" + metadata = RepoInfo( + name="test_repo", url="https://github.com/test/repo", hash="hash123" + ) + + link = DefaultNeedLink() + link.file = Path("file.py") + link.line = 0 + + result = get_github_link_from_json(metadata, link) + + assert result == "https://github.com/test/repo/blob/hash123/file.py#L0" + + +# Tests for get_github_link +def test_get_github_link_with_hash(): + """Test get_github_link uses json method when hash is present.""" + metadata = RepoInfo( + name="some_repo", url="https://github.com/org/repo", hash="commit_hash_123" + ) + + link = DefaultNeedLink() + link.file = Path("src/example.py") + link.line = 42 + + result = get_github_link(metadata, link) + + assert ( + result == "https://github.com/org/repo/blob/commit_hash_123/src/example.py#L42" + ) + + +@pytest.fixture +def temp_dir() -> Generator[Path, None, None]: + """Create a temporary directory for tests.""" + with tempfile.TemporaryDirectory() as temp_dir: + yield Path(temp_dir) + + +@pytest.fixture +def git_repo(temp_dir: Path) -> Path: + """Create a real git repository for testing.""" + git_dir: Path = temp_dir / "test_repo" + git_dir.mkdir() + + # Initialize git repo + _ = subprocess.run(["git", "init"], cwd=git_dir, check=True, capture_output=True) + _ = subprocess.run( + ["git", "config", "user.email", "test@example.com"], cwd=git_dir, check=True + ) + _ = subprocess.run( + ["git", "config", "user.name", "Test User"], cwd=git_dir, check=True + ) + + # Create a test file and commit + test_file: Path = git_dir / "test_file.py" + _ = test_file.write_text("# Test file\nprint('hello')\n") + _ = subprocess.run(["git", "add", "."], cwd=git_dir, check=True) + _ = subprocess.run( + ["git", "commit", "-m", "Initial commit"], cwd=git_dir, check=True + ) + + # Add a remote + _ = subprocess.run( + ["git", "remote", "add", "origin", "git@github.com:test-user/test-repo.git"], + cwd=git_dir, + check=True, + ) + return git_dir + + +def test_get_github_link_with_real_repo(git_repo: Path) -> None: + """ + Test generating GitHub link without url/hash. + This expects to be in a git repo + """ + metadata = RepoInfo(name="some_repo", url="", hash="") + + link = DefaultNeedLink() + link.file = Path("src/example.py") + link.line = 42 + + # Have to change directories in order to ensure that we get the right/any .git file + os.chdir(Path(git_repo).absolute()) + # ADAPTED: Using get_github_link_from_git for direct local repo testing + result = get_github_link(metadata, link) + + # Should contain the base URL, hash, file path, and line number + assert "https://github.com/test-user/test-repo/blob/" in result + assert "src/example.py#L42" in result + + # Verify the hash is actually from the repo + hash_from_link = result.split("/blob/")[1].split("/")[0] + actual_hash = get_current_git_hash(git_repo) + assert hash_from_link == actual_hash + + + +def test_complete_workflow(known_good_json: Path): + """Test complete workflow from path to GitHub link.""" + + # Parse path + path = Path("external/score_docs_as_code+/src/helper_lib/test_helper_lib.py") + repo_name = parse_repo_name_from_path(path) + assert repo_name == "score_docs_as_code" + + # Get metadata + hash_val, repo_url = parse_info_from_known_good(known_good_json, repo_name) + assert hash_val == "c1207676afe6cafd25c35d420e73279a799515d8" + assert repo_url == "https://github.com/eclipse-score/docs-as-code" + + # Generate link + metadata = RepoInfo(name=repo_name, url=repo_url, hash=hash_val) + link = DefaultNeedLink() + link.file = Path("src/helper_lib/test_helper_lib.py") + link.line = 75 + + result = get_github_link(metadata, link) + + assert ( + result + == "https://github.com/eclipse-score/docs-as-code/blob/c1207676afe6cafd25c35d420e73279a799515d8/src/helper_lib/test_helper_lib.py#L75" + ) diff --git a/src/extensions/score_source_code_linker/tests/test_repo_source_link_integration.py b/src/extensions/score_source_code_linker/tests/test_repo_source_link_integration.py new file mode 100644 index 000000000..79360e902 --- /dev/null +++ b/src/extensions/score_source_code_linker/tests/test_repo_source_link_integration.py @@ -0,0 +1,467 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# โ•“ โ•– +# โ•‘ Some portions generated by CoPilot โ•‘ +# โ•™ โ•œ + +import contextlib +import json +import os +import shutil +import subprocess +from collections.abc import Callable +from pathlib import Path + +import pytest +from pytest import TempPathFactory +from sphinx.testing.util import SphinxTestApp + +from src.extensions.score_source_code_linker.repo_source_links import ( + RepoInfo, + RepoSourceLinks_JSON_Decoder, + load_repo_source_links_json, +) + +""" + โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€INFORMATIONโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +# โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ +# โ”‚ boiler plate generated โ”‚ +# โ”‚ Human screened and adapted โ”‚ +# โ”‚ though still be aware of mistakes โ”‚ +# โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ + +""" + + +@pytest.fixture(scope="session") +def sphinx_base_dir(tmp_path_factory: TempPathFactory) -> Path: + return tmp_path_factory.mktemp("test_module_links_repo") + + +@pytest.fixture(scope="session") +def git_repo_setup(sphinx_base_dir: Path) -> Path: + """Creating git repo, to make testing possible""" + repo_path = sphinx_base_dir + _ = subprocess.run(["git", "init"], cwd=repo_path, check=True) + _ = subprocess.run( + ["git", "config", "user.name", "Test User"], cwd=repo_path, check=True + ) + _ = subprocess.run( + ["git", "config", "user.email", "test@example.com"], cwd=repo_path, check=True + ) + _ = subprocess.run( + ["git", "remote", "add", "origin", "https://github.com/testorg/modulerepo.git"], + cwd=repo_path, + check=True, + ) + os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(repo_path) + return repo_path + + +@pytest.fixture(scope="session") +def create_demo_files(sphinx_base_dir: Path, git_repo_setup: Path): + repo_path = sphinx_base_dir + + # Create source files + source_dir = repo_path / "src" + source_dir.mkdir() + + _ = (source_dir / "repo_a_impl.py").write_text(make_repo_a_source()) + _ = (source_dir / "repo_b_impl.py").write_text(make_repo_b_source()) + + # Create docs directory + docs_dir = repo_path / "docs" + docs_dir.mkdir() + _ = (docs_dir / "index.rst").write_text(basic_needs()) + _ = (docs_dir / "conf.py").write_text(basic_conf()) + + # Create test.xml files + bazel_testdir = repo_path / "bazel-testlogs" / "src" + bazel_testdir.mkdir(parents=True) + _ = (bazel_testdir / "test.xml").write_text(make_test_xml()) + + curr_dir = Path(__file__).absolute().parent + _ = shutil.copyfile( + curr_dir / "expected_repo_grouped.json", + repo_path / ".expected_repo_grouped.json", + ) + + # Commit everything + _ = subprocess.run(["git", "add", "."], cwd=repo_path, check=True) + _ = subprocess.run( + ["git", "commit", "-m", "Initial commit with module test files"], + cwd=repo_path, + check=True, + ) + + +def make_repo_a_source(): + return ( + """ +# Repo A implementation +# """ + + """req-Id: MOD_REQ_1 +def repo_a_function(): + pass + +# """ + + """req-Id: MOD_REQ_2 +class RepoAClass: + pass +""" + ) + + +def make_repo_b_source(): + return ( + """ +# Repo B implementation +# """ + + """req-Id: MOD_REQ_1 +def repo_b_function(): + pass + +# """ + + """req-Id: MOD_REQ_3 +def another_repo_b_function(): + pass +""" + ) + + +def make_test_xml(): + # ruff: noqa: E501 (start) + return """ + + + + + + + + + + + + + + + + + + +""" + # ruff: noqa: E501 (finish) + + +def basic_conf(): + return """ +extensions = [ + "sphinx_needs", + "score_source_code_linker", +] +needs_types = [ + dict( + directive="mod_req", + title="Repo Requirement", + prefix="MOD_REQ_", + color="#BFD8D2", + style="node", + ), +] +needs_extra_options = ["source_code_link", "testlink"] +needs_extra_links = [{ + "option": "partially_verifies", + "incoming": "paritally_verified_by", + "outgoing": "paritally_verifies", + }, + { + "option": "fully_verifies", + "incoming": "fully_verified_by", + "outgoing": "fully_verifies", + }] +""" + + +def basic_needs(): + return """ +MODULE TESTING +============== + +.. mod_req:: Repo Requirement 1 + :id: MOD_REQ_1 + :status: valid + +.. mod_req:: Repo Requirement 2 + :id: MOD_REQ_2 + :status: open + +.. mod_req:: Repo Requirement 3 + :id: MOD_REQ_3 + :status: open +""" + + +@pytest.fixture() +def sphinx_app_setup( + sphinx_base_dir: Path, create_demo_files: None, git_repo_setup: Path +) -> Callable[[], SphinxTestApp]: + def _create_app(): + base_dir = sphinx_base_dir + docs_dir = base_dir / "docs" + + original_cwd = None + with contextlib.suppress(FileNotFoundError): + original_cwd = os.getcwd() + + os.chdir(base_dir) + try: + return SphinxTestApp( + freshenv=True, + srcdir=docs_dir, + confdir=docs_dir, + outdir=sphinx_base_dir / "out", + buildername="html", + warningiserror=True, + ) + finally: + if original_cwd is not None: + with contextlib.suppress(FileNotFoundError, OSError): + os.chdir(original_cwd) + + return _create_app + + +def test_repo_grouped_cache_generated( + sphinx_app_setup: Callable[[], SphinxTestApp], + sphinx_base_dir: Path, + git_repo_setup: Path, + create_demo_files: None, +): + """Happy path: Repo grouped cache file is generated after Sphinx build""" + app = sphinx_app_setup() + try: + os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) + app.build() + + repo_cache = app.outdir / "score_repo_grouped_scl_cache.json" + assert repo_cache.exists(), "Repo grouped cache was not created" + + # Load and verify structure + loaded = load_repo_source_links_json(repo_cache) + assert isinstance(loaded, list) + assert len(loaded) > 0, "Repo cache should contain at least one repo" + + # Verify each item is a RepoSourceLinks + for item in loaded: + assert hasattr(item, "repo") + assert hasattr(item, "needs") + assert isinstance(item.repo, RepoInfo) + + finally: + app.cleanup() + + +def test_repo_grouping_preserves_metadata( + sphinx_app_setup: Callable[[], SphinxTestApp], + sphinx_base_dir: Path, + git_repo_setup: Path, + create_demo_files: None, +): + """Happy path: Repo metadata (name, hash, url) is preserved in cache""" + app = sphinx_app_setup() + try: + os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) + app.build() + + repo_cache = app.outdir / "score_repo_grouped_scl_cache.json" + loaded = load_repo_source_links_json(repo_cache) + + # Verify that each module has proper metadata + for repo_links in loaded: + assert repo_links.repo.name is not None + assert isinstance(repo_links.repo.name, str) + # Hash and URL might be empty strings for local module + assert repo_links.repo.hash is not None + assert repo_links.repo.url is not None + + finally: + app.cleanup() + + +def test_repo_grouping_multiple_needs_per_repo( + sphinx_app_setup: Callable[[], SphinxTestApp], + sphinx_base_dir: Path, + git_repo_setup: Path, + create_demo_files: None, +): + """Happy path: Multiple needs from same repo are grouped together""" + app = sphinx_app_setup() + try: + os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) + app.build() + + repo_cache = app.outdir / "score_repo_grouped_scl_cache.json" + loaded = load_repo_source_links_json(repo_cache) + + # Find the local_repo (should have all 3 requirements) + local_repo = None + for m in loaded: + if m.repo.name == "local_repo": + local_repo = m + break + + assert local_repo is not None, "local_repo not found in grouped cache" + + # All 3 MOD_REQ should be in this repo + need_ids = {need.need for need in local_repo.needs} + assert {"MOD_REQ_1", "MOD_REQ_2", "MOD_REQ_3"}.issubset(need_ids) + + finally: + app.cleanup() + + +def test_repo_cache_json_format( + sphinx_app_setup: Callable[[], SphinxTestApp], + sphinx_base_dir: Path, + git_repo_setup: Path, + create_demo_files: None, +): + """ + Repo cache JSON has correct + structure and excludes metadata from links + """ + app = sphinx_app_setup() + try: + os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) + app.build() + + repo_cache = app.outdir / "score_repo_grouped_scl_cache.json" + + # Load as raw JSON to check structure + with open(repo_cache) as f: + raw_json = json.load(f) + + assert isinstance(raw_json, list) + assert len(raw_json) > 0 + + # Check first repo structure + first_repo = raw_json[0] + assert "repo" in first_repo + assert "needs" in first_repo + assert "name" in first_repo["repo"] + assert "hash" in first_repo["repo"] + assert "url" in first_repo["repo"] + + # Check that needlinks don't have metadata + if first_repo["needs"]: + first_need = first_repo["needs"][0] + if "links" in first_need and first_need["links"].get("CodeLinks"): + codelink = first_need["links"]["CodeLinks"][0] + assert "repo_name" not in codelink, ( + "CodeLinks should not contain repo_name metadata" + ) + assert "hash" not in codelink, ( + "CodeLinks should not contain hash metadata" + ) + assert "url" not in codelink, ( + "CodeLinks should not contain url metadata" + ) + + finally: + app.cleanup() + + +def test_repo_cache_rebuilds_when_missing( + sphinx_app_setup: Callable[[], SphinxTestApp], + sphinx_base_dir: Path, + git_repo_setup: Path, + create_demo_files: None, +): + """Edge case: Repo cache is regenerated if deleted""" + app = sphinx_app_setup() + try: + os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) + app.build() + + repo_cache = app.outdir / "score_repo_grouped_scl_cache.json" + assert repo_cache.exists() + + # Delete the cache + repo_cache.unlink() + assert not repo_cache.exists() + + # Build again - should regenerate + app2 = sphinx_app_setup() + app2.build() + + assert repo_cache.exists(), "Cache should be regenerated on rebuild" + + # Verify it's valid + loaded = load_repo_source_links_json(repo_cache) + assert len(loaded) > 0 + + app2.cleanup() + finally: + app.cleanup() + + +def test_repo_grouping_with_golden_file( + sphinx_app_setup: Callable[[], SphinxTestApp], + sphinx_base_dir: Path, + git_repo_setup: Path, + create_demo_files: None, +): + """Happy path: Generated repo cache matches expected golden file""" + app = sphinx_app_setup() + try: + os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) + app.build() + + repo_cache = app.outdir / "score_repo_grouped_scl_cache.json" + expected_file = sphinx_base_dir / ".expected_repo_grouped.json" + + assert repo_cache.exists() + assert expected_file.exists(), "Golden file not found" + + with open(repo_cache) as f1: + actual = json.load(f1, object_hook=RepoSourceLinks_JSON_Decoder) + with open(expected_file) as f2: + expected = json.load(f2, object_hook=RepoSourceLinks_JSON_Decoder) + + assert len(actual) == len(expected), ( + f"Repo count mismatch. Actual: {len(actual)}, Expected: {len(expected)}" + ) + + # Compare repo by repo + actual_by_name = {m.repo.name: m for m in actual} + expected_by_name = {m.repo.name: m for m in expected} + + assert set(actual_by_name.keys()) == set(expected_by_name.keys()), ( + f"Repo names don't match. " + f"Actual: {set(actual_by_name.keys())}, " + f"Expected: {set(expected_by_name.keys())}" + ) + + for repo_name in actual_by_name: + actual_repo = actual_by_name[repo_name] + expected_repo = expected_by_name[repo_name] + + assert actual_repo.repo.hash == expected_repo.repo.hash + assert actual_repo.repo.url == expected_repo.repo.url + assert len(actual_repo.needs) == len(expected_repo.needs) + + finally: + app.cleanup() diff --git a/src/extensions/score_source_code_linker/tests/test_repo_source_links.py b/src/extensions/score_source_code_linker/tests/test_repo_source_links.py new file mode 100644 index 000000000..33a3d0384 --- /dev/null +++ b/src/extensions/score_source_code_linker/tests/test_repo_source_links.py @@ -0,0 +1,561 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import json +from dataclasses import asdict +from pathlib import Path +from typing import Any + +import pytest + +from src.extensions.score_source_code_linker.need_source_links import ( + NeedSourceLinks, + SourceCodeLinks, +) +from src.extensions.score_source_code_linker.needlinks import NeedLink +from src.extensions.score_source_code_linker.repo_source_links import ( + RepoInfo, + RepoSourceLinks, + RepoSourceLinks_JSON_Decoder, + group_needs_by_repo, + load_repo_source_links_json, + store_repo_source_links_json, +) +from src.extensions.score_source_code_linker.testlink import DataForTestLink + +""" + โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€INFORMATIONโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +# โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ +# โ”‚ partially generated โ”‚ +# โ”‚ Human screened and adapted โ”‚ +# โ”‚ though still be aware of mistakes โ”‚ +# โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ + +""" + + +# โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ +# โ”‚ JSON ENCODER TEST โ”‚ +# โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ + + +def encode_comment(s: str) -> str: + return s.replace(" ", "-----", 1) + + +def decode_comment(s: str) -> str: + return s.replace("-----", " ", 1) + + +def SourceCodeLinks_TEST_JSON_Decoder( + d: dict[str, Any], +) -> SourceCodeLinks | dict[str, Any]: + if "need" in d and "links" in d: + links = d["links"] + + # Decode CodeLinks + code_links = [] + for cl in links.get("CodeLinks", []): + # Decode the tag and full_line fields + if "tag" in cl: + cl["tag"] = decode_comment(cl["tag"]) + if "full_line" in cl: + cl["full_line"] = decode_comment(cl["full_line"]) + code_links.append(NeedLink(**cl)) + + # Decode TestLinks + return SourceCodeLinks( + need=d["need"], + links=NeedSourceLinks( + CodeLinks=code_links, + TestLinks=[DataForTestLink(**tl) for tl in links.get("TestLinks", [])], + ), + ) + return d + + +class repoSourceLinks_TEST_JSON_Encoder(json.JSONEncoder): + def default(self, o: object) -> str | dict[str, Any]: + if isinstance(o, Path): + return str(o) + # We do not want to save the metadata inside the codelink or testlink + # As we save this already in a structure above it + # (hash, repo_name, url) + if isinstance(o, NeedLink | DataForTestLink): + d = o.to_dict_without_metadata() + tag = d.get("tag", "") + full_line = d.get("full_line", "") + assert isinstance(tag, str) + assert isinstance(full_line, str) + d["tag"] = encode_comment(tag) + d["full_line"] = encode_comment(full_line) + return d + # We need to split this up, otherwise the nested + # dictionaries won't get split up and we will not + # run into the 'to_dict_without_metadata' as + # everything will be converted to a normal dictionary + if isinstance(o, RepoSourceLinks): + return { + "repo": asdict(o.repo), + "needs": o.needs, # Let the encoder handle the list + } + if isinstance(o, SourceCodeLinks): + return { + "need": o.need, + "links": o.links, + } + if isinstance(o, NeedSourceLinks): + return { + "CodeLinks": o.CodeLinks, + "TestLinks": o.TestLinks, + } + return super().default(o) + + +def repoSourceLinks_TEST_JSON_Decoder( + d: dict[str, Any], +) -> RepoSourceLinks | dict[str, Any]: + if "repo" in d and "needs" in d: + repo = d["repo"] + needs = d["needs"] + return RepoSourceLinks( + repo=RepoInfo( + name=repo.get("name"), + hash=repo.get("hash"), + url=repo.get("url"), + ), + # We know this can only be list[SourceCodeLinks] and nothing else + # Therefore => we ignore the type error here + needs=[SourceCodeLinks_TEST_JSON_Decoder(need) for need in needs], # type: ignore + ) + return d + + +def test_json_encoder_removes_metadata_from_needlink(): + """Happy path: NeedLink metadata fields are excluded from JSON output""" + encoder = repoSourceLinks_TEST_JSON_Encoder() + needlink = NeedLink( + file=Path("src/test.py"), + line=10, + tag="#" + " req-Id:", + need="REQ_1", + full_line="#" + " req-Id: REQ_1", + repo_name="test_repo", + url="https://github.com/test/repo", + hash="abc123", + ) + result = encoder.default(needlink) + + assert isinstance(result, dict) + assert "repo_name" not in result + assert "url" not in result + assert "hash" not in result + assert result["need"] == "REQ_1" + assert result["line"] == 10 + + +def test_json_encoder_removes_metadata_from_testlink(): + """Happy path: DataForTestLink metadata fields are excluded from JSON output""" + encoder = repoSourceLinks_TEST_JSON_Encoder() + testlink = DataForTestLink( + name="test_something", + file=Path("src/test_file.py"), + need="REQ_1", + line=20, + verify_type="fully", + result="passed", + result_text="", + repo_name="test_repo", + url="https://github.com/test/repo", + hash="abc123", + ) + result = encoder.default(testlink) + + assert isinstance(result, dict) + assert "repo_name" not in result + assert "url" not in result + assert "hash" not in result + assert result["name"] == "test_something" + assert result["need"] == "REQ_1" + + +def test_json_encoder_converts_path_to_string(): + """Happy path: Path objects are converted to strings""" + encoder = repoSourceLinks_TEST_JSON_Encoder() + result = encoder.default(Path("/test/path/file.py")) + assert result == "/test/path/file.py" + assert isinstance(result, str) + + +# ============================================================================ +# JSON Decoder Tests +# ============================================================================ + + +def test_json_decoder_reconstructs_repo_source_links(): + """Happy path: Valid JSON dict is decoded into repoSourceLinks""" + json_data: dict[str, Any] = { + "repo": {"name": "test_repo", "hash": "hash1", "url": "url1"}, + "needs": [ + { + "need": "REQ_1", + "links": {"CodeLinks": [], "TestLinks": []}, + } + ], + } + result = RepoSourceLinks_JSON_Decoder(json_data) + + assert isinstance(result, RepoSourceLinks) + assert result.repo.name == "test_repo" + assert result.repo.hash == "hash1" + assert result.repo.url == "url1" + assert len(result.needs) == 1 + + +def test_json_decoder_returns_unchanged_for_non_repo_dict(): + """Edge case: Non-repoSourceLinks dicts are returned unchanged""" + json_data = {"some": "data", "other": "values"} + result = RepoSourceLinks_JSON_Decoder(json_data) + assert result == json_data + + +# ============================================================================ +# Store/Load Tests +# ============================================================================ + + +def test_store_and_load_roundtrip(tmp_path: Path): + """Happy path: Store and load preserves data correctly""" + repo = RepoInfo(name="test_repo", hash="hash1", url="url1") + needlink = NeedLink( + file=Path("src/test.py"), + line=10, + tag="#" + " req-Id:", + need="REQ_1", + full_line="#" + " req-Id: REQ_1", + repo_name="test_repo", + url="url1", + hash="hash1", + ) + scl = SourceCodeLinks( + need="REQ_1", + links=NeedSourceLinks(CodeLinks=[needlink], TestLinks=[]), + ) + repo_links = RepoSourceLinks(repo=repo, needs=[scl]) + + test_file = tmp_path / "test.json" + store_repo_source_links_json(test_file, [repo_links]) + loaded = load_repo_source_links_json(test_file) + + assert len(loaded) == 1 + assert loaded[0].repo.name == "test_repo" + assert loaded[0].needs[0].need == "REQ_1" + + +def test_store_creates_parent_directories(tmp_path: Path): + """Edge case: Parent directories are created if they don't exist""" + nested_path = tmp_path / "nested" / "deeply" / "test.json" + repo = RepoInfo(name="test", hash="h", url="u") + repo_links = RepoSourceLinks(repo=repo, needs=[]) + + store_repo_source_links_json(nested_path, [repo_links]) + + assert nested_path.exists() + assert nested_path.parent.exists() + + +def test_load_empty_list(tmp_path: Path): + """Edge case: Loading empty list returns empty list""" + test_file = tmp_path / "empty.json" + store_repo_source_links_json(test_file, []) + + loaded = load_repo_source_links_json(test_file) + assert loaded == [] + + +def test_load_validates_is_list(tmp_path: Path): + """Bad path: Loading non-list JSON fails validation""" + test_file = tmp_path / "not_list.json" + test_file.write_text('{"repo": {}, "needs": []}') + + with pytest.raises(AssertionError, match="should be a list"): + load_repo_source_links_json(test_file) + + +def test_load_validates_items_are_correct_type(tmp_path: Path): + """Bad path: Loading list with invalid items fails validation""" + test_file = tmp_path / "invalid_items.json" + test_file.write_text('[{"invalid": "structure"}]') + + with pytest.raises(AssertionError, match="should be RepoSourceLink objects"): + load_repo_source_links_json(test_file) + + +# ============================================================================ +# group_needs_by_repo Tests +# ============================================================================ + + +def test_group_needs_single_repo_with_codelinks(): + """Happy path: Multiple needs from same repo are grouped together""" + needlink1 = NeedLink( + file=Path("src/file1.py"), + line=10, + tag="#" + " req-Id:", + need="REQ_1", + full_line="#" + " req-Id: REQ_1", + repo_name="shared_repo", + url="https://github.com/test/repo", + hash="hash1", + ) + needlink2 = NeedLink( + file=Path("src/file2.py"), + line=20, + tag="#" + " req-Id:", + need="REQ_2", + full_line="#" + " req-Id: REQ_2", + repo_name="shared_repo", + url="https://github.com/test/repo", + hash="hash1", + ) + + scl1 = SourceCodeLinks( + need="REQ_1", links=NeedSourceLinks(CodeLinks=[needlink1], TestLinks=[]) + ) + scl2 = SourceCodeLinks( + need="REQ_2", links=NeedSourceLinks(CodeLinks=[needlink2], TestLinks=[]) + ) + + result = group_needs_by_repo([scl1, scl2]) + + assert len(result) == 1 + assert result[0].repo.name == "shared_repo" + assert result[0].repo.hash == "hash1" + assert len(result[0].needs) == 2 + assert len(result[0].needs[0].links.CodeLinks) == 1 + assert len(result[0].needs[1].links.CodeLinks) == 1 + + +def test_group_needs_multiple_repos(): + """Happy path: Needs from different repos create separate groups""" + needlink_a = NeedLink( + file=Path("src/a.py"), + line=10, + tag="#" + " req-Id:", + need="REQ_1", + full_line="#" + " req-Id: REQ_1", + repo_name="repo_a", + url="https://github.com/a/repo", + hash="hash_a", + ) + needlink_b = NeedLink( + file=Path("src/b.py"), + line=20, + tag="#" + " req-Id:", + need="REQ_2", + full_line="#" + " req-Id: REQ_2", + repo_name="repo_b", + url="https://github.com/b/repo", + hash="hash_b", + ) + + scl1 = SourceCodeLinks( + need="REQ_1", links=NeedSourceLinks(CodeLinks=[needlink_a], TestLinks=[]) + ) + scl2 = SourceCodeLinks( + need="REQ_2", links=NeedSourceLinks(CodeLinks=[needlink_b], TestLinks=[]) + ) + + result = group_needs_by_repo([scl1, scl2]) + + assert len(result) == 2 + assert result[0].repo.name == "repo_a" + assert result[0].repo.hash == "hash_a" + assert result[0].repo.url == "https://github.com/a/repo" + assert result[1].repo.name == "repo_b" + assert result[1].repo.hash == "hash_b" + assert result[1].repo.url == "https://github.com/b/repo" + + +def test_group_needs_with_testlinks_only(): + """Happy path: Needs with only test links (no code links) are grouped correctly""" + testlink = DataForTestLink( + name="test_feature", + file=Path("tests/test.py"), + need="REQ_1", + line=15, + verify_type="fully", + result="passed", + result_text="", + repo_name="test_repo", + url="https://github.com/test/repo", + hash="hash1", + ) + testlink2 = DataForTestLink( + name="test_feature", + file=Path("tests/test2.py"), + need="REQ_10", + line=153, + verify_type="partially", + result="passed", + result_text="", + repo_name="test_repo", + url="https://github.com/test/repo", + hash="hash1", + ) + + scl = SourceCodeLinks( + need="REQ_1", + links=NeedSourceLinks(CodeLinks=[], TestLinks=[testlink]), + ) + scl2 = SourceCodeLinks( + need="REQ_10", + links=NeedSourceLinks(CodeLinks=[], TestLinks=[testlink2]), + ) + + result = group_needs_by_repo([scl, scl2]) + + assert len(result) == 1 + assert result[0].repo.name == "test_repo" + assert len(result[0].needs) == 2 + needs = [x.need for x in result[0].needs] + assert "REQ_1" in needs + assert "REQ_10" in needs + assert len(result[0].needs[0].links.TestLinks) == 1 + assert len(result[0].needs[1].links.TestLinks) == 1 + + +def test_group_needs_with_testlinks_different_repos(): + """Testing Testlinks grouping for different repos""" + testlink = DataForTestLink( + name="test_feature", + file=Path("tests/test.py"), + need="REQ_1", + line=15, + verify_type="fully", + result="passed", + result_text="", + repo_name="repo_a", + url="https://github.com/test_a/repo_a", + hash="hash_a", + ) + testlink2 = DataForTestLink( + name="test_feature", + file=Path("tests/test3.py"), + need="REQ_10", + line=153, + verify_type="partially", + result="passed", + result_text="", + repo_name="repo_b", + url="https://github.com/test_b/repo_b", + hash="hash_b", + ) + + scl = SourceCodeLinks( + need="REQ_1", + links=NeedSourceLinks(CodeLinks=[], TestLinks=[testlink]), + ) + scl2 = SourceCodeLinks( + need="REQ_10", + links=NeedSourceLinks(CodeLinks=[], TestLinks=[testlink2]), + ) + + result = group_needs_by_repo([scl, scl2]) + + assert len(result) == 2 + assert result[0].repo.name == "repo_a" + assert result[0].repo.hash == "hash_a" + assert result[0].repo.url == "https://github.com/test_a/repo_a" + assert result[1].repo.name == "repo_b" + assert result[1].repo.hash == "hash_b" + assert result[1].repo.url == "https://github.com/test_b/repo_b" + + +def test_group_needs_empty_list(): + """Edge case: Empty list returns empty result""" + result = group_needs_by_repo([]) + assert result == [] + + +def test_group_needs_skips_needs_without_links(): + """ + Edge case: Needs with no code or test links are skipped + This should normally never even be possible to get + to this function. Though we should test what happens + """ + scl_with_links = SourceCodeLinks( + need="REQ_1", + links=NeedSourceLinks( + CodeLinks=[ + NeedLink( + file=Path("src/test.py"), + line=10, + tag="#" + " req-Id:", + need="REQ_1", + full_line="#" + " req-Id: REQ_1", + repo_name="repo_a", + url="url1", + hash="hash1", + ) + ], + TestLinks=[], + ), + ) + scl_without_links = SourceCodeLinks( + need="REQ_2", + links=NeedSourceLinks(CodeLinks=[], TestLinks=[]), + ) + + result = group_needs_by_repo([scl_with_links, scl_without_links]) + + assert len(result) == 1 + assert result[0].needs[0].need == "REQ_1" + assert result[0].needs[0].links.CodeLinks[0].full_line == "#" + " req-Id: REQ_1" + + +def test_group_needs_mixed_codelinks_and_testlinks(): + """Happy path: Needs with both code and test links are handled correctly""" + needlink = NeedLink( + file=Path("src/impl.py"), + line=5, + tag="#" + " req-Id:", + need="REQ_1", + full_line="#" + " req-Id: REQ_1", + repo_name="repo_a", + url="https://github.com/test/repo", + hash="hash1", + ) + testlink = DataForTestLink( + name="test_impl", + file=Path("tests/test_impl.py"), + need="REQ_1", + line=10, + verify_type="fully", + result="passed", + result_text="", + repo_name="repo_a", + url="https://github.com/test/repo", + hash="hash1", + ) + + scl = SourceCodeLinks( + need="REQ_1", + links=NeedSourceLinks(CodeLinks=[needlink], TestLinks=[testlink]), + ) + + result = group_needs_by_repo([scl]) + + assert len(result) == 1 + assert len(result[0].needs[0].links.CodeLinks) == 1 + assert len(result[0].needs[0].links.TestLinks) == 1 diff --git a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py index 60bb98f80..9d604a43a 100644 --- a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py +++ b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py @@ -25,7 +25,9 @@ from sphinx.testing.util import SphinxTestApp from sphinx_needs.data import SphinxNeedsData +from src.extensions.score_source_code_linker.helpers import get_github_link from src.extensions.score_source_code_linker.needlinks import NeedLink +from src.extensions.score_source_code_linker.repo_source_links import RepoInfo from src.extensions.score_source_code_linker.testlink import ( DataForTestLink, DataForTestLink_JSON_Decoder, @@ -37,15 +39,14 @@ SourceCodeLinks_TEST_JSON_Decoder, ) from src.helper_lib import find_ws_root, get_github_base_url -from src.helper_lib.additional_functions import get_github_link -@pytest.fixture() +@pytest.fixture(scope="session") def sphinx_base_dir(tmp_path_factory: TempPathFactory) -> Path: return tmp_path_factory.mktemp("test_git_repo") -@pytest.fixture() +@pytest.fixture(scope="session") def git_repo_setup(sphinx_base_dir: Path) -> Path: """Creating git repo, to make testing possible""" @@ -67,7 +68,7 @@ def git_repo_setup(sphinx_base_dir: Path) -> Path: return repo_path -@pytest.fixture() +@pytest.fixture(scope="session") def create_demo_files(sphinx_base_dir: Path, git_repo_setup: Path): repo_path = sphinx_base_dir @@ -312,6 +313,9 @@ def example_source_link_text_all_ok(sphinx_base_dir: Path) -> dict[str, list[Nee tag="#" + " req-Id:", need="TREQ_ID_1", full_line="#" + " req-Id: TREQ_ID_1", + repo_name="local_repo", + url="", + hash="", ), NeedLink( file=Path("src/implementation1.py"), @@ -319,6 +323,9 @@ def example_source_link_text_all_ok(sphinx_base_dir: Path) -> dict[str, list[Nee tag="#" + " req-Id:", need="TREQ_ID_1", full_line="#" + " req-Id: TREQ_ID_1", + repo_name="local_repo", + url="", + hash="", ), ], "TREQ_ID_2": [ @@ -328,6 +335,9 @@ def example_source_link_text_all_ok(sphinx_base_dir: Path) -> dict[str, list[Nee tag="#" + " req-Id:", need="TREQ_ID_2", full_line="#" + " req-Id: TREQ_ID_2", + repo_name="local_repo", + url="", + hash="", ) ], "TREQ_ID_3": [], @@ -346,6 +356,9 @@ def example_test_link_text_all_ok(sphinx_base_dir: Path): verify_type="fully", result="passed", result_text="", + repo_name="local_repo", + url="", + hash="", ), ], "TREQ_ID_2": [ @@ -357,6 +370,9 @@ def example_test_link_text_all_ok(sphinx_base_dir: Path): verify_type="partially", result="passed", result_text="", + repo_name="local_repo", + url="", + hash="", ), DataForTestLink( name="test_error_handling", @@ -366,6 +382,9 @@ def example_test_link_text_all_ok(sphinx_base_dir: Path): verify_type="partially", result="passed", result_text="", + repo_name="local_repo", + url="", + hash="", ), ], "TREQ_ID_3": [ @@ -377,6 +396,9 @@ def example_test_link_text_all_ok(sphinx_base_dir: Path): verify_type="partially", result="passed", result_text="", + repo_name="local_repo", + url="", + hash="", ), DataForTestLink( name="test_error_handling", @@ -386,6 +408,9 @@ def example_test_link_text_all_ok(sphinx_base_dir: Path): verify_type="partially", result="passed", result_text="", + repo_name="local_repo", + url="", + hash="", ), ], } @@ -409,11 +434,23 @@ def example_source_link_text_non_existent(sphinx_base_dir: Path): def make_source_link(needlinks: list[NeedLink]): - return ", ".join(f"{get_github_link(n)}<>{n.file}:{n.line}" for n in needlinks) + metadata = RepoInfo( + name="local_repo", + url="", + hash="", + ) + return ", ".join( + f"{get_github_link(metadata, n)}<>{n.file}:{n.line}" for n in needlinks + ) def make_test_link(testlinks: list[DataForTestLink]): - return ", ".join(f"{get_github_link(n)}<>{n.name}" for n in testlinks) + metadata = RepoInfo( + name="local_repo", + url="", + hash="", + ) + return ", ".join(f"{get_github_link(metadata, n)}<>{n.name}" for n in testlinks) def compare_json_files( diff --git a/src/extensions/score_source_code_linker/tests/test_testlink.py b/src/extensions/score_source_code_linker/tests/test_testlink.py index 74becef6b..9d3e8ecab 100644 --- a/src/extensions/score_source_code_linker/tests/test_testlink.py +++ b/src/extensions/score_source_code_linker/tests/test_testlink.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2025 Contributors to the Eclipse Foundation +# Copyright (c) 2026 Contributors to the Eclipse Foundation # # See the NOTICE file(s) distributed with this work for additional # information regarding copyright ownership. @@ -10,9 +10,16 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* + +# โ•“ โ•– +# โ•‘ Some portions generated by Co-Pilot โ•‘ +# โ•™ โ•œ + import json from pathlib import Path +import pytest + # This depends on the `attribute_plugin` in our tooling repository from attribute_plugin import add_test_properties # type: ignore[import-untyped] @@ -21,7 +28,11 @@ DataForTestLink_JSON_Decoder, DataForTestLink_JSON_Encoder, DataOfTestCase, + DataOfTestCase_JSON_Decoder, + DataOfTestCase_JSON_Encoder, + load_data_of_test_case_json, load_test_xml_parsed_json, + store_data_of_test_case_json, store_test_xml_parsed_json, ) @@ -41,6 +52,10 @@ def test_testlink_serialization_roundtrip(): verify_type="fully", result="passed", result_text="All good", + # ADAPTED: Added new fields + repo_name="test_repo", + hash="abc12345", + url="https://github.com/org/repo", ) dumped = json.dumps(link, cls=DataForTestLink_JSON_Encoder) loaded = json.loads(dumped, object_hook=DataForTestLink_JSON_Decoder) @@ -101,6 +116,10 @@ def test_testcaseneed_to_dict_multiple_links(): result_text="Something went wrong", PartiallyVerifies="REQ-1, REQ-2", FullyVerifies="REQ-3", + # ADAPTED: Added new fields which are now required for valid TestLinks + repo_name="test_repo", + hash="hash123", + url="http://github.com", ) links = case.get_test_links() @@ -114,6 +133,10 @@ def test_testcaseneed_to_dict_multiple_links(): assert link.line == 10 assert link.name == "TC_01" assert link.result == "failed" + # ADAPTED: Verify new fields are propagated + assert link.repo_name == "test_repo" + assert link.hash == "hash123" + assert link.url == "http://github.com" @add_test_properties( @@ -134,6 +157,10 @@ def test_store_and_load_testlinks_roundtrip(tmp_path: Path): verify_type="partially", result="passed", result_text="Looks good", + # ADAPTED: Added new fields + repo_name="mod1", + hash="h1", + url="u1", ), DataForTestLink( name="L2", @@ -143,6 +170,10 @@ def test_store_and_load_testlinks_roundtrip(tmp_path: Path): verify_type="fully", result="failed", result_text="Needs work", + # ADAPTED: Added new fields + repo_name="mod2", + hash="h2", + url="u2", ), ] @@ -154,3 +185,221 @@ def test_store_and_load_testlinks_roundtrip(tmp_path: Path): assert reloaded == links for link in reloaded: assert isinstance(link, DataForTestLink) + + +def test_datafortestlink_to_dict_full(): + """Cover line 85: to_dict_full includes all fields""" + link = DataForTestLink( + name="test_full", + file=Path("src/test.py"), + line=10, + need="REQ_1", + verify_type="fully", + result="passed", + result_text="All good", + repo_name="test_repo", + hash="abc123", + url="https://github.com/test/repo", + ) + result = link.to_dict_full() + + assert result["repo_name"] == "test_repo" + assert result["hash"] == "abc123" + assert result["url"] == "https://github.com/test/repo" + assert result["name"] == "test_full" + + +def test_datafortestlink_equality_with_non_testlink(): + """Cover line 69: __eq__ with non-DataForTestLink returns NotImplemented""" + link = DataForTestLink( + name="test", + file=Path("src/test.py"), + line=10, + need="REQ_1", + verify_type="fully", + result="passed", + ) + + result = link.__eq__("not a testlink") + assert result is NotImplemented + + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€[ DataOfTestCase ]โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +def test_dataoftestcase_check_verifies_fields_missing_both(): + """Cover lines 200-206: check_verifies_fields returns False when both are None""" + case = DataOfTestCase( + name="TC_Missing", + file="src/test.py", + line="10", + result="passed", + TestType="unit", + DerivationTechnique="manual", + result_text="", + PartiallyVerifies=None, + FullyVerifies=None, + repo_name="mod", + hash="h", + url="u", + ) + + assert case.check_verifies_fields() is False + + +def test_dataoftestcase_is_valid_fails_on_none_field(): + """Cover lines 232-238: is_valid returns False when required field is None""" + case = DataOfTestCase( + name="TC_Invalid", + file=None, # Missing required field + line="10", + result="passed", + TestType="unit", + DerivationTechnique="manual", + result_text="", + PartiallyVerifies="REQ_1", + repo_name="mod", + hash="h", + url="u", + ) + + assert case.is_valid() is False + + +def test_dataoftestcase_get_test_links_returns_empty_when_invalid(): + """Cover line 245: get_test_links returns [] when is_valid is False""" + case = DataOfTestCase( + name="TC_Invalid", + file=None, # Invalid + line="10", + result="passed", + TestType="unit", + DerivationTechnique="manual", + PartiallyVerifies="REQ_1", + ) + + links = case.get_test_links() + assert links == [] + + +def test_dataoftestcase_decoder_valid_dict(): + """Cover lines 314-324: Decoder reconstructs DataOfTestCase from valid dict""" + json_data = { + "name": "TC_01", + "file": "src/test.py", + "line": "10", + "result": "passed", + "TestType": "unit", + "DerivationTechnique": "manual", + "result_text": "Good", + "PartiallyVerifies": "REQ_1", + "FullyVerifies": "REQ_2", + "repo_name": "mod", + "hash": "h", + "url": "u", + } + result = DataOfTestCase_JSON_Decoder(json_data) + + assert isinstance(result, DataOfTestCase) + assert result.name == "TC_01" + assert result.file == "src/test.py" + + +def test_dataoftestcase_decoder_non_testcase_dict(): + """Cover line 326: Decoder returns dict unchanged for non-TestCase data""" + json_data = {"some": "other", "data": "here"} + result = DataOfTestCase_JSON_Decoder(json_data) + assert result == json_data + + +def test_store_and_load_data_of_test_case_roundtrip(tmp_path: Path): + """Cover lines 376-386: store and load DataOfTestCase roundtrip""" + test_cases = [ + DataOfTestCase( + name="TC_A", + file="src/a.py", + line="5", + result="passed", + TestType="unit", + DerivationTechnique="manual", + result_text="OK", + PartiallyVerifies="REQ_A", + FullyVerifies=None, + repo_name="mod_a", + hash="hash_a", + url="url_a", + ), + DataOfTestCase( + name="TC_B", + file="src/b.py", + line="15", + result="failed", + TestType="integration", + DerivationTechnique="auto", + result_text="Failed", + PartiallyVerifies=None, + FullyVerifies="REQ_B", + repo_name="mod_b", + hash="hash_b", + url="url_b", + ), + ] + + test_file = tmp_path / "test_cases.json" + store_data_of_test_case_json(test_file, test_cases) + loaded = load_data_of_test_case_json(test_file) + + assert len(loaded) == 2 + assert loaded[0].name == "TC_A" + assert loaded[1].name == "TC_B" + + +def test_load_data_of_test_case_validates_list(tmp_path: Path): + """Cover line 380: Load validates result is a list""" + test_file = tmp_path / "not_list.json" + test_file.write_text('{"name": "TC_01"}') + + with pytest.raises(AssertionError, match="should be a list"): + _ = load_data_of_test_case_json(test_file) + + +def test_load_data_of_test_case_validates_items(tmp_path: Path): + """Cover line 383: Load validates all items are DataOfTestCase""" + test_file = tmp_path / "invalid_items.json" + test_file.write_text('[{"invalid": "structure"}]') + + with pytest.raises(AssertionError, match="should be TestCaseNeed objects"): + _ = load_data_of_test_case_json(test_file) + + +def test_dataoftestcase_encoder_fallback(): + """Cover line 299: Encoder falls back to parent for unknown types""" + encoder = DataOfTestCase_JSON_Encoder() + + # This should trigger the super().default() call + with pytest.raises(TypeError): + _ = encoder.default(object()) + + +def test_datafortestlink_encoder_fallback(): + """Cover line 103: DataForTestLink encoder falls back to parent for unknown types""" + from src.extensions.score_source_code_linker.testlink import ( + DataForTestLink_JSON_Encoder, + ) + + encoder = DataForTestLink_JSON_Encoder() + + # This should trigger the super().default() call + with pytest.raises(TypeError): + _ = encoder.default(object()) + + +def test_datafortestlink_decoder_non_testlink_dict(): + """Cover line 132: DataForTestLink decoder returns dict unchanged""" + from src.extensions.score_source_code_linker.testlink import ( + DataForTestLink_JSON_Decoder, + ) + + json_data = {"random": "data", "other": "stuff"} + result = DataForTestLink_JSON_Decoder(json_data) + assert result == json_data diff --git a/src/extensions/score_source_code_linker/tests/test_xml_parser.py b/src/extensions/score_source_code_linker/tests/test_xml_parser.py index 95b445dde..3a2e110c7 100644 --- a/src/extensions/score_source_code_linker/tests/test_xml_parser.py +++ b/src/extensions/score_source_code_linker/tests/test_xml_parser.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2025 Contributors to the Eclipse Foundation +# Copyright (c) 2026 Contributors to the Eclipse Foundation # # See the NOTICE file(s) distributed with this work for additional # information regarding copyright ownership. @@ -10,6 +10,11 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* + +# โ•“ โ•– +# โ•‘ Some portions generated by CoPilot โ•‘ +# โ•™ โ•œ + """ Tests for the xml_parser.py file. Keep in mind that this is with the 'assertions' inside xml_parser disabled so far. @@ -20,6 +25,7 @@ from collections.abc import Callable from pathlib import Path from typing import Any +from unittest.mock import patch import pytest @@ -262,10 +268,18 @@ def test_parse_properties(): test_type="requirements-based", derivation_technique="requirements-analysis", ) +# ADAPTED: Added patching for metadata functions +@patch("src.extensions.score_source_code_linker.xml_parser.parse_repo_name_from_path") def test_read_test_xml_file( + mock_parse_module: Any, tmp_xml_dirs: Callable[..., tuple[Path, Path, Path, Path, Path]], ): """Ensure a whole pre-defined xml file is parsed correctly""" + # ADAPTED: Mock return value to ensure metadata is populated. + # 'local_module' triggers the path where hash/url are empty strings, + # which is valid but avoids the need to mock parse_info_from_known_good. + mock_parse_module.return_value = "local_repo" + _: Path dir1: Path dir2: Path @@ -276,6 +290,11 @@ def test_read_test_xml_file( tcneed = needs1[0] assert isinstance(tcneed, DataOfTestCase) assert tcneed.result == "failed" + # ADAPTED: Verify metadata fields were populated + assert tcneed.repo_name == "local_repo" + assert tcneed.hash == "" + assert tcneed.url == "" + assert no_props1 == [] assert missing_props1 == [] @@ -287,7 +306,7 @@ def test_read_test_xml_file( # Extra Properties => Should not cause an error needs3, no_props3, missing_props3 = xml_parser.read_test_xml_file(dir3 / "test.xml") - assert isinstance(needs1, list) and len(needs1) == 1 + assert isinstance(needs3, list) and len(needs3) == 1 tcneed3 = needs3[0] assert isinstance(tcneed3, DataOfTestCase) assert no_props3 == [] @@ -312,3 +331,54 @@ def test_short_hash_consistency_and_format(): assert h1 == h2 assert h1.isalpha() assert len(h1) == 5 + + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€[ Boilerplate generated by CoPilot ]โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +def test_clean_test_file_name_local_path(): + raw_path = Path( + "/home/root/folder/docs-as-code/bazel-testlogs/src/extensions/score_any_folder/score_any_folder_tests/test.xml" + ) + result = xml_parser.clean_test_file_name(raw_path) + assert result == Path( + "src/extensions/score_any_folder/score_any_folder_tests/test.xml" + ) + + +def test_clean_test_file_name_combo_path(): + raw_path = Path( + "/root/bazel-testlogs/external/score_docs_as_code+/src/extensions/score_any_folder/score_any_folder_tests/test.xml" + ) + result = xml_parser.clean_test_file_name(raw_path) + assert result == Path( + "external/score_docs_as_code+/src/extensions/score_any_folder/score_any_folder_tests/test.xml" + ) + + +def test_clean_test_file_name_tests_report_path(): + raw_path = Path("/some/path/tests-report/test-report/unit/test_module.xml") + result = xml_parser.clean_test_file_name(raw_path) + assert result == Path("test-report/unit/test_module.xml") + + +def test_clean_test_file_name_nested_bazel_testlogs(): + raw_path = Path("/deeply/nested/bazel-testlogs/bazel-testlogs/final/path.xml") + result = xml_parser.clean_test_file_name(raw_path) + assert result == Path("final/path.xml") + + +def test_clean_test_file_name_invalid_path_raises_error(): + raw_path = Path("/invalid/path/without/markers/test.xml") + with pytest.raises( + ValueError, match="Filepath does not have 'bazel-testlogs' nor 'tests-report'" + ): + xml_parser.clean_test_file_name(raw_path) + + +def test_clean_test_file_name_empty_path_raises_error(): + raw_path = Path("") + with pytest.raises( + ValueError, match="Filepath does not have 'bazel-testlogs' nor 'tests-report'" + ): + xml_parser.clean_test_file_name(raw_path) diff --git a/src/extensions/score_source_code_linker/xml_parser.py b/src/extensions/score_source_code_linker/xml_parser.py index 8432e1fc3..cfa41f7f3 100644 --- a/src/extensions/score_source_code_linker/xml_parser.py +++ b/src/extensions/score_source_code_linker/xml_parser.py @@ -33,18 +33,94 @@ from sphinx_needs import logging from sphinx_needs.api import add_external_need +from src.extensions.score_source_code_linker.helpers import ( + get_github_link, + parse_info_from_known_good, + parse_repo_name_from_path, +) +from src.extensions.score_source_code_linker.needlinks import ( + DefaultMetaData, + MetaData, +) +from src.extensions.score_source_code_linker.repo_source_links import RepoInfo from src.extensions.score_source_code_linker.testlink import ( DataOfTestCase, store_data_of_test_case_json, store_test_xml_parsed_json, ) from src.helper_lib import find_ws_root -from src.helper_lib.additional_functions import get_github_link logger = logging.get_logger(__name__) logger.setLevel("DEBUG") +def clean_test_file_name(raw_filepath: Path) -> Path: + """ + incoming path: + `local` + /docs-as-code/bazel-testlogs/src/extensions/score_any_folder/score_any_folder_tests/test.xml + `combo` + /bazel-testlogs/external/score_docs_as_code+/src/extensions/score_any_folder/score_any_folder_tests/test.xml + + outgoing path: + `local` + src/extensions/score_any_folder/score_any_folder_tests/test.xml + `combo` + external/score_docs_as_code+/src/extensions/score_any_folder/score_any_folder_tests/test.xml + """ + if "bazel-testlogs" in str(raw_filepath): + return Path(str(raw_filepath).split("bazel-testlogs/")[-1]) + if "tests-report" in str(raw_filepath): + return Path(str(raw_filepath).split("tests-report/")[-1]) + raise ValueError( + "Filepath does not have 'bazel-testlogs' nor " + + f"'tests-report'. Filepath: {raw_filepath}" + ) + + +def get_metadata_from_test_path(raw_filepath: Path) -> MetaData: + """ + Will parse out the metadata from the testpath. + If test is local then the metadata will be: + + "repo_name": "local_repo", + "hash": "", + "url": "", + + Else it will parse the repo_name e.g. `score_docs_as_code` + match this in the known_good_json and grab the accompanying + hash, url as well and return metadata like so for example: + + "repo_name": "score_docs_as_code", + "hash": "c1207676afe6cafd25c35d420e73279a799515d8", + "url": "https://github.com/eclipse-score/docs-as-code" + + The file name passed into here is: + + For combo builds: something like: + /bazel-testlogs/external/score_docs_as_code+/ + src/extensions/score_any_folder/score_any_folder_tests/test.xml + + For local builds: + /bazel-testlogs/src/ext/score_.../score_any_folder_tests/test.xml + + Therefore will we 'clean' it before passing it to the parse_module func. + Removing everything up to and including 'bazel-testlogs' or 'tests-report' + """ + # print("THIs IS FILEPATH IN GET MD FROm TestPATH: ", raw_filepath) + known_good_json = os.environ.get("KNOWN_GOOD_JSON") + clean_filepath = clean_test_file_name(raw_filepath) + # print(f"This is the cleaned filepath: {clean_filepath}") + repo_name = parse_repo_name_from_path(clean_filepath) + md = DefaultMetaData() + md["repo_name"] = repo_name + if repo_name != "local_repo" and known_good_json: + md["hash"], md["url"] = parse_info_from_known_good( + Path(known_good_json), repo_name + ) + return md + + def parse_testcase_result(testcase: ET.Element) -> tuple[str, str]: """ Returns 'result' and 'result_text' found in the 'message' @@ -101,7 +177,7 @@ def read_test_xml_file(file: Path) -> tuple[list[DataOfTestCase], list[str], lis missing_prop_tests: list[str] = [] tree = ET.parse(file) root = tree.getroot() - + md = get_metadata_from_test_path(file) for testsuite in root.findall("testsuite"): for testcase in testsuite.findall("testcase"): case_properties = {} @@ -161,6 +237,7 @@ def read_test_xml_file(file: Path) -> tuple[list[DataOfTestCase], list[str], lis # If the is_valid method would return 'False' anyway. # I just can't think of it right now, leaving this for future me case_properties = parse_properties(case_properties, properties_element) + case_properties.update(md) test_case = DataOfTestCase.from_dict(case_properties) if not test_case.is_valid(): missing_prop_tests.append(testname) @@ -175,6 +252,11 @@ def find_xml_files(dir: Path) -> list[Path]: Returns: - list[Path] => Paths to all found 'test.xml' files. + + Example combo TestPath for future reference: + + '/reference_integration/bazel-testlogs + /feature_integration_tests/test_cases/fit/test.xml' """ test_file_name = "test.xml" @@ -183,6 +265,7 @@ def find_xml_files(dir: Path) -> list[Path]: for root, _, files in os.walk(dir): if test_file_name in files: xml_paths.append(Path(os.path.join(root, test_file_name))) + return xml_paths @@ -204,7 +287,6 @@ def run_xml_parser(app: Sphinx, env: BuildEnvironment): It gets called from the source_code_linker __init__ """ testlogs_dir = find_test_folder() - # early return if testlogs_dir is None: return xml_file_paths = find_xml_files(testlogs_dir) @@ -262,6 +344,11 @@ def construct_and_add_need(app: Sphinx, tn: DataOfTestCase): # and either 'Fully' or 'PartiallyVerifies' should not be None here assert tn.file is not None assert tn.name is not None + assert tn.repo_name is not None + assert tn.hash is not None + assert tn.url is not None + # Have to build metadata here for the gh link func + metadata = RepoInfo(name=tn.repo_name, hash=tn.hash, url=tn.url) # IDK if this is ideal or not with contextlib.suppress(BaseException): _ = add_external_need( @@ -269,9 +356,9 @@ def construct_and_add_need(app: Sphinx, tn: DataOfTestCase): need_type="testcase", title=tn.name, tags="TEST", - id=f"testcase__{tn.name}_{short_hash(tn.file + tn.name).upper()}", + id=f"testcase__{tn.name}_{short_hash(tn.file + tn.name)}", name=tn.name, - external_url=get_github_link(tn), + external_url=get_github_link(metadata, tn), fully_verifies=tn.FullyVerifies if tn.FullyVerifies is not None else "", partially_verifies=tn.PartiallyVerifies if tn.PartiallyVerifies is not None diff --git a/src/helper_lib/BUILD b/src/helper_lib/BUILD index 748a2a730..ad6316363 100644 --- a/src/helper_lib/BUILD +++ b/src/helper_lib/BUILD @@ -27,8 +27,7 @@ py_library( visibility = ["//visibility:public"], deps = [ "@rules_python//python/runfiles", - "@score_docs_as_code//src/extensions/score_source_code_linker:source_code_linker_helpers", - ], + ] + all_requirements, ) score_py_pytest( diff --git a/src/helper_lib/__init__.py b/src/helper_lib/__init__.py index 5699e478d..a72fffb0b 100644 --- a/src/helper_lib/__init__.py +++ b/src/helper_lib/__init__.py @@ -17,7 +17,7 @@ from pathlib import Path from typing import Any -from runfiles import Runfiles +from python.runfiles import Runfiles from sphinx.config import Config from sphinx_needs.logging import get_logger diff --git a/src/helper_lib/additional_functions.py b/src/helper_lib/additional_functions.py deleted file mode 100644 index 5b1ce6d98..000000000 --- a/src/helper_lib/additional_functions.py +++ /dev/null @@ -1,38 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2025 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -from pathlib import Path - -# Import types that depend on score_source_code_linker -from src.extensions.score_source_code_linker.needlinks import DefaultNeedLink, NeedLink -from src.extensions.score_source_code_linker.testlink import ( - DataForTestLink, - DataOfTestCase, -) -from src.helper_lib import ( - find_git_root, - get_current_git_hash, - get_github_base_url, -) - - -def get_github_link( - link: NeedLink | DataForTestLink | DataOfTestCase | None = None, -) -> str: - if link is None: - link = DefaultNeedLink() - passed_git_root = find_git_root() - if passed_git_root is None: - passed_git_root = Path() - base_url = get_github_base_url() - current_hash = get_current_git_hash(passed_git_root) - return f"{base_url}/blob/{current_hash}/{link.file}#L{link.line}" diff --git a/src/incremental.py b/src/incremental.py index 1c3816229..fa1c5abc2 100644 --- a/src/incremental.py +++ b/src/incremental.py @@ -90,6 +90,8 @@ def get_env(name: str) -> str: base_arguments.append(f"-A=github_repo={args.github_repo}") base_arguments.append("-A=github_version=main") base_arguments.append(f"-A=doc_path={get_env('SOURCE_DIRECTORY')}") + if os.getenv("KNOWN_GOOD_JSON"): + base_arguments.append(f"--define=KNOWN_GOOD_JSON={get_env('KNOWN_GOOD_JSON')}") action = get_env("ACTION") if action == "live_preview":