Fix script to handle SARIF file recategorization

.github/workflows/codeql-multiple-repo-scan.yml

@@ -54,11 +54,15 @@ jobs:
       - name: Parse known_good.json and create repos.json
         id: parse-repos
         run: |
-          scripts/workflow/parse_repos.sh
+          python3 scripts/workflow/parse_repos.py
       - name: Checkout all pinned repositories
         id: checkout-repos
         run: |
-          scripts/workflow/checkout_repos.sh
+          python3 scripts/workflow/checkout_repos.py
+      - name: List files in repos directory (debug)
+        run: |
+          echo "Listing all files in repos directory before CodeQL analysis:"
+          find repos || echo "repos directory not found"
       - name: Initialize CodeQL for all repositories
         uses: github/codeql-action/init@v4
         with:
@@ -75,7 +79,7 @@ jobs:
       - name: Recategorize Guidelines
         if: always()
         run: |
-          scripts/workflow/recategorize_guidelines.sh
+          python3 scripts/workflow/recategorize_guidelines.py
       - name: Generate HTML Report from SARIF
         run: |
           SARIF_FILE="sarif-results/cpp.sarif"

scripts/workflow/checkout_repos.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+# *******************************************************************************
+# Copyright (c) 2025 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+"""
+Checkout all pinned repositories based on repos.json configuration.
+"""
+
+import json
+import sys
+import subprocess
+import re
+import os
+from pathlib import Path
+
+
+def load_repos_config(config_file="./repos.json"):
+    """
+    Load repository configuration from repos.json.
+
+    Args:
+        config_file: Path to repos.json file
+
+    Returns:
+        List of repository configurations
+    """
+    config_path = Path(config_file)
+
+    if not config_path.exists():
+        print(f"Error: file not found '{config_file}'", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        with open(config_path, 'r') as f:
+            repos = json.load(f)
+        return repos
+    except (json.JSONDecodeError, IOError) as e:
+        print(f"Error: Failed to load repos.json: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def is_commit_hash(ref):
+    """
+    Check if reference looks like a commit hash (40 hex characters for SHA-1).
+
+    Args:
+        ref: Git reference (branch, tag, or hash)
+
+    Returns:
+        True if ref matches commit hash pattern
+    """
+    return bool(re.match(r'^[0-9a-fA-F]{40}$', ref))
+
+
+def checkout_repo(name, url, ref, path):
+    """
+    Checkout a single repository.
+
+    Args:
+        name: Repository name
+        url: Repository URL
+        ref: Git reference (branch, tag, or commit hash)
+        path: Local path to checkout into
+
+    Returns:
+        True if successful, False otherwise
+    """
+    path_obj = Path(path)
+
+    try:
+        # Create parent directory if needed
+        path_obj.parent.mkdir(parents=True, exist_ok=True)
+
+        if is_commit_hash(ref):
+            print(f"Checking out {name} ({ref}) to {path}")
+            print(f"  Detected commit hash. Cloning and then checking out.")
+
+            # Clone the repository
+            subprocess.run(
+                ["git", "clone", url, path],
+                check=True,
+                capture_output=True
+            )
+
+            # Checkout specific commit
+            subprocess.run(
+                ["git", "-C", path, "checkout", ref],
+                check=True,
+                capture_output=True
+            )
+        else:
+            print(f"Checking out {name} ({ref}) to {path}")
+            print(f"  Detected branch/tag. Cloning with --branch.")
+
+            # Clone with shallow copy and specific branch/tag
+            # Add 'v' prefix if not already present (common convention)
+            branch_ref = ref if ref.startswith('v') else f'v{ref}'
+            subprocess.run(
+                ["git", "clone", "--depth", "1", "--branch", branch_ref, url, path],
+                check=True,
+                capture_output=True
+            )
+
+        return True
+
+    except subprocess.CalledProcessError as e:
+        print(f"Error: Failed to checkout {name}: {e}", file=sys.stderr)
+        return False
+
+
+def main():
+    """Main entry point."""
+    # Load repository configurations
+    repos = load_repos_config('./repos.json')
+    repo_count = len(repos)
+
+    # Track successfully checked out repositories
+    repo_paths = []
+
+    # Checkout each repository
+    for i, repo in enumerate(repos):
+        name = repo.get('name', f'repo-{i}')
+        url = repo.get('url', '')
+        ref = repo.get('version', '')
+        path = repo.get('path', '')
+
+        if not all([url, ref, path]):
+            print(f"Warning: Skipping {name} - missing required fields", file=sys.stderr)
+            continue
+
+        if checkout_repo(name, url, ref, path):
+            repo_paths.append(path)
+
+    # Output all paths (comma-separated for GitHub Actions compatibility)
+    repo_paths_output = ','.join(repo_paths)
+
+    # Write to GITHUB_OUTPUT if available
+    github_output = os.environ.get('GITHUB_OUTPUT')
+    if github_output:
+        try:
+            with open(github_output, 'a') as f:
+                f.write(f"repo_paths={repo_paths_output}\n")
+        except IOError as e:
+            print(f"Warning: Failed to write GITHUB_OUTPUT: {e}", file=sys.stderr)
+
+    # Also print for debugging
+    print(f"\nSuccessfully checked out {len(repo_paths)} of {repo_count} repositories")
+    print(f"repo_paths={repo_paths_output}")
+
+    return 0 if len(repo_paths) == repo_count else 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())

scripts/workflow/parse_repos.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+# *******************************************************************************
+# Copyright (c) 2025 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+"""
+Parse known_good.json and create repos.json for multi-repository CodeQL analysis.
+"""
+
+import json
+import sys
+import subprocess
+from pathlib import Path
+
+
+def install_dependencies():
+    """Ensure jq is installed (for reference, though we use Python's json)."""
+    try:
+        subprocess.run(
+            ["sudo", "apt-get", "update"],
+            check=True,
+            capture_output=True
+        )
+        subprocess.run(
+            ["sudo", "apt-get", "install", "-y", "jq"],
+            check=True,
+            capture_output=True
+        )
+    except subprocess.CalledProcessError as e:
+        print(f"Warning: Failed to install jq: {e}", file=sys.stderr)
+
+
+def parse_known_good(json_file="./known_good.json"):
+    """
+    Parse known_good.json and transform modules into repository objects.
+
+    Args:
+        json_file: Path to known_good.json file
+
+    Returns:
+        Tuple of (repos list, module count, module outputs dict)
+    """
+    json_path = Path(json_file)
+
+    if not json_path.exists():
+        print(f"Error: file not found '{json_file}'", file=sys.stderr)
+        print(f"Current directory: {Path.cwd()}", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        with open(json_path, 'r') as f:
+            data = json.load(f)
+    except json.JSONDecodeError as e:
+        print(f"Error: Failed to parse JSON: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    # Extract target_sw modules
+    modules = data.get('modules', {}).get('target_sw', {})
+
+    # Transform modules into repository objects
+    repos = []
+    module_outputs = {}
+
+    for name, config in modules.items():
+        repo_url = config.get('repo', '')
+        version = config.get('version', '')
+        branch = config.get('branch', '')
+        hash_val = config.get('hash', '')
+
+        # Use version, branch, or hash (in that order of preference)
+        ref = version or branch or hash_val
+
+        repo_obj = {
+            'name': name,
+            'url': repo_url,
+            'version': ref,
+            'path': f'repos/{name}'
+        }
+        repos.append(repo_obj)
+
+        # Store module outputs for GITHUB_OUTPUT compatibility
+        module_outputs[f'{name}_url'] = repo_url
+        if version:
+            module_outputs[f'{name}_version'] = version
+        if branch:
+            module_outputs[f'{name}_branch'] = branch
+        if hash_val:
+            module_outputs[f'{name}_hash'] = hash_val
+
+    return repos, len(modules), module_outputs
+
+
+def write_repos_json(repos, output_file="./repos.json"):
+    """Write repositories to repos.json file."""
+    output_path = Path(output_file)
+
+    try:
+        with open(output_path, 'w') as f:
+            json.dump(repos, f, indent=2)
+        print(f"Generated {output_file}:")
+        print(json.dumps(repos, indent=2))
+        print()  # Add newline for readability
+    except IOError as e:
+        print(f"Error: Failed to write {output_file}: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def write_github_output(outputs):
+    """
+    Write outputs to GITHUB_OUTPUT for GitHub Actions compatibility.
+
+    Args:
+        outputs: Dictionary of key-value pairs to output
+    """
+    github_output = Path(os.environ.get('GITHUB_OUTPUT', '/dev/null'))
+
+    if github_output.exists() or github_output.parent.exists():
+        try:
+            with open(github_output, 'a') as f:
+                for key, value in outputs.items():
+                    f.write(f"{key}={value}\n")
+        except IOError as e:
+            print(f"Warning: Failed to write GITHUB_OUTPUT: {e}", file=sys.stderr)
+
+
+def main():
+    """Main entry point."""
+    import os
+
+    # Install dependencies (optional, jq not strictly needed in Python version)
+    # install_dependencies()
+
+    # Parse known_good.json
+    repos, module_count, module_outputs = parse_known_good('./known_good.json')
+
+    # Write repos.json
+    write_repos_json(repos)
+
+    # Write GitHub Actions outputs
+    github_outputs = {'MODULE_COUNT': str(module_count)}
+    github_outputs.update(module_outputs)
+    write_github_output(github_outputs)
+
+    print("Parse complete!")
+
+
+if __name__ == '__main__':
+    main()