From 3e75b4e2aa770c90196595359d5d387e7e959cd5 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Wed, 28 Jan 2026 17:52:34 +0000 Subject: [PATCH 1/8] build: add script to determine interdependencies between modules in the monorepo --- .kokoro/determine_dependencies.py | 270 ++++++++++++++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 .kokoro/determine_dependencies.py diff --git a/.kokoro/determine_dependencies.py b/.kokoro/determine_dependencies.py new file mode 100644 index 000000000000..eb90b6c2cc1d --- /dev/null +++ b/.kokoro/determine_dependencies.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import sys +import xml.etree.ElementTree as ET +from collections import defaultdict +from typing import Dict, List, Set, Tuple + +# Maven XML namespace +NS = {'mvn': 'http://maven.apache.org/POM/4.0.0'} + +class Module: + def __init__(self, path: str, group_id: str, artifact_id: str, parent: Tuple[str, str] = None): + self.path = path + self.group_id = group_id + self.artifactId = artifact_id + self.parent = parent + self.dependencies: Set[Tuple[str, str]] = set() + + @property + def key(self) -> Tuple[str, str]: + return (self.group_id, self.artifactId) + + def __repr__(self): + return f"{self.group_id}:{self.artifactId}" + +def parse_pom(path: str) -> Module: + try: + tree = ET.parse(path) + root = tree.getroot() + except ET.ParseError as e: + print(f"Error parsing {path}: {e}", file=sys.stderr) + return None + + # Handle namespace if present + # XML tags in ElementTree are {namespace}tag + # We'll use find with namespaces for robustness, but simple logic for extraction + + # Helper to clean tag name + def local_name(tag): + if '}' in tag: + return tag.split('}', 1)[1] + return tag + + parent_elem = root.find('mvn:parent', NS) + parent_coords = None + parent_group_id = None + if parent_elem is not None: + p_group = parent_elem.find('mvn:groupId', NS).text + p_artifact = parent_elem.find('mvn:artifactId', NS).text + parent_coords = (p_group, p_artifact) + parent_group_id = p_group + + group_id_elem = root.find('mvn:groupId', NS) + # Inherit groupId from parent if not specified + if group_id_elem is not None: + group_id = group_id_elem.text + elif parent_group_id: + group_id = parent_group_id + else: + # Fallback or error? For now, use artifactId as heuristic or empty + group_id = "unknown" + + artifact_id = root.find('mvn:artifactId', NS).text + + module = Module(path, group_id, artifact_id, parent_coords) + + # Dependencies + def add_dependencies(section): + if section is not None: + for dep in section.findall('mvn:dependency', NS): + d_group = dep.find('mvn:groupId', NS) + d_artifact = dep.find('mvn:artifactId', NS) + if d_group is not None and d_artifact is not None: + module.dependencies.add((d_group.text, d_artifact.text)) + + add_dependencies(root.find('mvn:dependencies', NS)) + + dep_mgmt = root.find('mvn:dependencyManagement', NS) + if dep_mgmt is not None: + add_dependencies(dep_mgmt.find('mvn:dependencies', NS)) + + # Plugin dependencies + build = root.find('mvn:build', NS) + if build is not None: + plugins = build.find('mvn:plugins', NS) + if plugins is not None: + for plugin in plugins.findall('mvn:plugin', NS): + # Plugin itself + p_group = plugin.find('mvn:groupId', NS) + p_artifact = plugin.find('mvn:artifactId', NS) + if p_group is not None and p_artifact is not None: + module.dependencies.add((p_group.text, p_artifact.text)) + + # Plugin dependencies + add_dependencies(plugin.find('mvn:dependencies', NS)) + + # Plugin Management + plugin_mgmt = build.find('mvn:pluginManagement', NS) + if plugin_mgmt is not None: + plugins = plugin_mgmt.find('mvn:plugins', NS) + if plugins is not None: + for plugin in plugins.findall('mvn:plugin', NS): + # Plugin itself + p_group = plugin.find('mvn:groupId', NS) + p_artifact = plugin.find('mvn:artifactId', NS) + if p_group is not None and p_artifact is not None: + module.dependencies.add((p_group.text, p_artifact.text)) + + add_dependencies(plugin.find('mvn:dependencies', NS)) + + return module + +def find_poms(root_dir: str) -> List[str]: + pom_files = [] + for dirpath, dirnames, filenames in os.walk(root_dir): + # Skip hidden directories and known non-module dirs + dirnames[:] = [d for d in dirnames if not d.startswith('.')] + + if 'pom.xml' in filenames: + pom_files.append(os.path.join(dirpath, 'pom.xml')) + return pom_files + +def build_graph(root_dir: str) -> Tuple[Dict[Tuple[str, str], Module], Dict[Tuple[str, str], Set[Tuple[str, str]]]]: + pom_paths = find_poms(root_dir) + modules: Dict[Tuple[str, str], Module] = {} + + # First pass: load all modules + for path in pom_paths: + module = parse_pom(path) + if module: + modules[module.key] = module + + # Build adjacency list: dependent -> dependencies (upstream) + # Only include dependencies that are present in the repo + graph: Dict[Tuple[str, str], Set[Tuple[str, str]]] = defaultdict(set) + + for key, module in modules.items(): + # Parent dependency + if module.parent and module.parent in modules: + graph[key].add(module.parent) + + # Regular dependencies + for dep_key in module.dependencies: + if dep_key in modules: + graph[key].add(dep_key) + + return modules, graph + +def get_transitive_dependencies( + start_nodes: List[Tuple[str, str]], + graph: Dict[Tuple[str, str], Set[Tuple[str, str]]] +) -> Set[Tuple[str, str]]: + visited = set() + stack = list(start_nodes) + + while stack: + node = stack.pop() + if node not in visited: + visited.add(node) + # Add upstream dependencies to stack + if node in graph: + for upstream in graph[node]: + if upstream not in visited: + stack.append(upstream) + + return visited + +def resolve_modules_from_inputs(inputs: List[str], modules_by_path: Dict[str, Module], modules_by_key: Dict[Tuple[str, str], Module]) -> List[Tuple[str, str]]: + resolved = set() + for item in inputs: + # Check if item is a path + abs_item = os.path.abspath(item) + + # If it's a file, try to find the nearest pom.xml + if os.path.isfile(abs_item) or (not item.endswith('pom.xml') and os.path.isdir(abs_item)): + # Heuristic: if it's a file, find containing pom + # if it's a dir, look for pom.xml inside or check if it matches a module path + candidate_path = abs_item + if os.path.isfile(candidate_path) and not candidate_path.endswith('pom.xml'): + candidate_path = os.path.dirname(candidate_path) + + # Traverse up to find pom.xml + while candidate_path.startswith(os.getcwd()) and len(candidate_path) >= len(os.getcwd()): + pom_path = os.path.join(candidate_path, 'pom.xml') + if pom_path in modules_by_path: + resolved.add(modules_by_path[pom_path].key) + break + candidate_path = os.path.dirname(candidate_path) + elif item.endswith('pom.xml') and os.path.abspath(item) in modules_by_path: + resolved.add(modules_by_path[os.path.abspath(item)].key) + else: + # Try to match simple name (artifactId) or groupId:artifactId + found = False + for key, module in modules_by_key.items(): + if item == module.artifactId or item == f"{module.group_id}:{module.artifactId}": + resolved.add(key) + found = True + break + if not found: + print(f"Warning: Could not resolve input '{item}' to a module.", file=sys.stderr) + + return list(resolved) + +def main(): + parser = argparse.ArgumentParser(description='Identify upstream dependencies for partial builds.') + parser.add_argument('modules', nargs='+', help='List of modified modules or file paths') + args = parser.parse_args() + + root_dir = os.getcwd() + modules_by_key, graph = build_graph(root_dir) + modules_by_path = {m.path: m for m in modules_by_key.values()} + + start_nodes = resolve_modules_from_inputs(args.modules, modules_by_path, modules_by_key) + + if not start_nodes: + print("No valid modules found from input.", file=sys.stderr) + return + + # Get transitive upstream dependencies + # We include the start nodes themselves in the output set if they are dependencies of other start nodes? + # Usually we want: Dependencies of (Start Nodes) NOT INCLUDING Start Nodes themselves, unless A depends on B and both are modified. + # But for "installing dependencies", we generally want EVERYTHING upstream of the modified set. + # If I modified A, and A depends on B, I want to install B. + # If I modified A and B, and A depends on B, I want to install B (before A). + # But usually the build system will build A and B if I say "build A and B". + # The request is: "determine which modules will need to be compiled and installed to the local maven repository" + # This implies we want the COMPLEMENT set of the modified modules, restricted to the upstream graph. + + all_dependencies = get_transitive_dependencies(start_nodes, graph) + + # Filter out the start nodes themselves, because they are the "modified" ones + # that will presumably be built by the test command itself? + # Actually, usually 'dependencies.sh' installs everything needed for the tests. + # If I am testing A, I need B installed. + # If I am testing A and B, I need C (upstream of B) installed. + # So I need (TransitiveClosure(StartNodes) - StartNodes). + + upstream_only = all_dependencies - set(start_nodes) + + # Map back to paths or artifactIds? + # `mvn -pl` takes directory paths or [groupId]:artifactId + # Directory paths are safer if artifactIds are not unique (rare but possible) + # relpath is good. + + results = [] + for key in upstream_only: + module = modules_by_key[key] + rel_path = os.path.relpath(os.path.dirname(module.path), root_dir) + # Maven -pl expects project dir or group:artifact + results.append(rel_path) + + print(','.join(sorted(results))) + +if __name__ == '__main__': + main() From ab6a9e73932f6a7e1995b33edad59c4039f1cc4d Mon Sep 17 00:00:00 2001 From: chingor13 Date: Wed, 28 Jan 2026 18:07:34 +0000 Subject: [PATCH 2/8] fix dependency order --- .kokoro/determine_dependencies.py | 233 ++++++++++++++++++------------ 1 file changed, 140 insertions(+), 93 deletions(-) diff --git a/.kokoro/determine_dependencies.py b/.kokoro/determine_dependencies.py index eb90b6c2cc1d..540f80a5d9eb 100644 --- a/.kokoro/determine_dependencies.py +++ b/.kokoro/determine_dependencies.py @@ -21,10 +21,13 @@ from typing import Dict, List, Set, Tuple # Maven XML namespace -NS = {'mvn': 'http://maven.apache.org/POM/4.0.0'} +NS = {"mvn": "http://maven.apache.org/POM/4.0.0"} + class Module: - def __init__(self, path: str, group_id: str, artifact_id: str, parent: Tuple[str, str] = None): + def __init__( + self, path: str, group_id: str, artifact_id: str, parent: Tuple[str, str] = None + ): self.path = path self.group_id = group_id self.artifactId = artifact_id @@ -38,6 +41,7 @@ def key(self) -> Tuple[str, str]: def __repr__(self): return f"{self.group_id}:{self.artifactId}" + def parse_pom(path: str) -> Module: try: tree = ET.parse(path) @@ -49,23 +53,23 @@ def parse_pom(path: str) -> Module: # Handle namespace if present # XML tags in ElementTree are {namespace}tag # We'll use find with namespaces for robustness, but simple logic for extraction - + # Helper to clean tag name def local_name(tag): - if '}' in tag: - return tag.split('}', 1)[1] + if "}" in tag: + return tag.split("}", 1)[1] return tag - parent_elem = root.find('mvn:parent', NS) + parent_elem = root.find("mvn:parent", NS) parent_coords = None parent_group_id = None if parent_elem is not None: - p_group = parent_elem.find('mvn:groupId', NS).text - p_artifact = parent_elem.find('mvn:artifactId', NS).text + p_group = parent_elem.find("mvn:groupId", NS).text + p_artifact = parent_elem.find("mvn:artifactId", NS).text parent_coords = (p_group, p_artifact) parent_group_id = p_group - group_id_elem = root.find('mvn:groupId', NS) + group_id_elem = root.find("mvn:groupId", NS) # Inherit groupId from parent if not specified if group_id_elem is not None: group_id = group_id_elem.text @@ -75,70 +79,80 @@ def local_name(tag): # Fallback or error? For now, use artifactId as heuristic or empty group_id = "unknown" - artifact_id = root.find('mvn:artifactId', NS).text - + artifact_id = root.find("mvn:artifactId", NS).text + module = Module(path, group_id, artifact_id, parent_coords) # Dependencies - def add_dependencies(section): + def add_dependencies(section, is_management=False): if section is not None: - for dep in section.findall('mvn:dependency', NS): - d_group = dep.find('mvn:groupId', NS) - d_artifact = dep.find('mvn:artifactId', NS) + for dep in section.findall("mvn:dependency", NS): + if is_management: + # Only include 'import' scope dependencies from dependencyManagement + scope = dep.find("mvn:scope", NS) + if scope is None or scope.text != "import": + continue + + d_group = dep.find("mvn:groupId", NS) + d_artifact = dep.find("mvn:artifactId", NS) if d_group is not None and d_artifact is not None: module.dependencies.add((d_group.text, d_artifact.text)) - add_dependencies(root.find('mvn:dependencies', NS)) - - dep_mgmt = root.find('mvn:dependencyManagement', NS) + add_dependencies(root.find("mvn:dependencies", NS)) + + dep_mgmt = root.find("mvn:dependencyManagement", NS) if dep_mgmt is not None: - add_dependencies(dep_mgmt.find('mvn:dependencies', NS)) + add_dependencies(dep_mgmt.find("mvn:dependencies", NS), is_management=True) # Plugin dependencies - build = root.find('mvn:build', NS) + build = root.find("mvn:build", NS) if build is not None: - plugins = build.find('mvn:plugins', NS) + plugins = build.find("mvn:plugins", NS) if plugins is not None: - for plugin in plugins.findall('mvn:plugin', NS): + for plugin in plugins.findall("mvn:plugin", NS): # Plugin itself - p_group = plugin.find('mvn:groupId', NS) - p_artifact = plugin.find('mvn:artifactId', NS) + p_group = plugin.find("mvn:groupId", NS) + p_artifact = plugin.find("mvn:artifactId", NS) if p_group is not None and p_artifact is not None: module.dependencies.add((p_group.text, p_artifact.text)) - + # Plugin dependencies - add_dependencies(plugin.find('mvn:dependencies', NS)) - + add_dependencies(plugin.find("mvn:dependencies", NS)) + # Plugin Management - plugin_mgmt = build.find('mvn:pluginManagement', NS) + plugin_mgmt = build.find("mvn:pluginManagement", NS) if plugin_mgmt is not None: - plugins = plugin_mgmt.find('mvn:plugins', NS) - if plugins is not None: - for plugin in plugins.findall('mvn:plugin', NS): + plugins = plugin_mgmt.find("mvn:plugins", NS) + if plugins is not None: + for plugin in plugins.findall("mvn:plugin", NS): # Plugin itself - p_group = plugin.find('mvn:groupId', NS) - p_artifact = plugin.find('mvn:artifactId', NS) + p_group = plugin.find("mvn:groupId", NS) + p_artifact = plugin.find("mvn:artifactId", NS) if p_group is not None and p_artifact is not None: module.dependencies.add((p_group.text, p_artifact.text)) - add_dependencies(plugin.find('mvn:dependencies', NS)) + add_dependencies(plugin.find("mvn:dependencies", NS)) return module + def find_poms(root_dir: str) -> List[str]: pom_files = [] for dirpath, dirnames, filenames in os.walk(root_dir): # Skip hidden directories and known non-module dirs - dirnames[:] = [d for d in dirnames if not d.startswith('.')] - - if 'pom.xml' in filenames: - pom_files.append(os.path.join(dirpath, 'pom.xml')) + dirnames[:] = [d for d in dirnames if not d.startswith(".")] + + if "pom.xml" in filenames: + pom_files.append(os.path.join(dirpath, "pom.xml")) return pom_files -def build_graph(root_dir: str) -> Tuple[Dict[Tuple[str, str], Module], Dict[Tuple[str, str], Set[Tuple[str, str]]]]: + +def build_graph( + root_dir: str, +) -> Tuple[Dict[Tuple[str, str], Module], Dict[Tuple[str, str], Set[Tuple[str, str]]]]: pom_paths = find_poms(root_dir) modules: Dict[Tuple[str, str], Module] = {} - + # First pass: load all modules for path in pom_paths: module = parse_pom(path) @@ -148,26 +162,27 @@ def build_graph(root_dir: str) -> Tuple[Dict[Tuple[str, str], Module], Dict[Tupl # Build adjacency list: dependent -> dependencies (upstream) # Only include dependencies that are present in the repo graph: Dict[Tuple[str, str], Set[Tuple[str, str]]] = defaultdict(set) - + for key, module in modules.items(): # Parent dependency if module.parent and module.parent in modules: graph[key].add(module.parent) - + # Regular dependencies for dep_key in module.dependencies: if dep_key in modules: graph[key].add(dep_key) - + return modules, graph + def get_transitive_dependencies( - start_nodes: List[Tuple[str, str]], - graph: Dict[Tuple[str, str], Set[Tuple[str, str]]] + start_nodes: List[Tuple[str, str]], + graph: Dict[Tuple[str, str], Set[Tuple[str, str]]], ) -> Set[Tuple[str, str]]: visited = set() stack = list(start_nodes) - + while stack: node = stack.pop() if node not in visited: @@ -177,62 +192,86 @@ def get_transitive_dependencies( for upstream in graph[node]: if upstream not in visited: stack.append(upstream) - + return visited -def resolve_modules_from_inputs(inputs: List[str], modules_by_path: Dict[str, Module], modules_by_key: Dict[Tuple[str, str], Module]) -> List[Tuple[str, str]]: + +def resolve_modules_from_inputs( + inputs: List[str], + modules_by_path: Dict[str, Module], + modules_by_key: Dict[Tuple[str, str], Module], +) -> List[Tuple[str, str]]: resolved = set() for item in inputs: # Check if item is a path abs_item = os.path.abspath(item) - + # If it's a file, try to find the nearest pom.xml - if os.path.isfile(abs_item) or (not item.endswith('pom.xml') and os.path.isdir(abs_item)): - # Heuristic: if it's a file, find containing pom - # if it's a dir, look for pom.xml inside or check if it matches a module path - candidate_path = abs_item - if os.path.isfile(candidate_path) and not candidate_path.endswith('pom.xml'): - candidate_path = os.path.dirname(candidate_path) - - # Traverse up to find pom.xml - while candidate_path.startswith(os.getcwd()) and len(candidate_path) >= len(os.getcwd()): - pom_path = os.path.join(candidate_path, 'pom.xml') - if pom_path in modules_by_path: - resolved.add(modules_by_path[pom_path].key) - break - candidate_path = os.path.dirname(candidate_path) - elif item.endswith('pom.xml') and os.path.abspath(item) in modules_by_path: - resolved.add(modules_by_path[os.path.abspath(item)].key) + if os.path.isfile(abs_item) or ( + not item.endswith("pom.xml") and os.path.isdir(abs_item) + ): + # Heuristic: if it's a file, find containing pom + # if it's a dir, look for pom.xml inside or check if it matches a module path + candidate_path = abs_item + if os.path.isfile(candidate_path) and not candidate_path.endswith( + "pom.xml" + ): + candidate_path = os.path.dirname(candidate_path) + + # Traverse up to find pom.xml + while candidate_path.startswith(os.getcwd()) and len(candidate_path) >= len( + os.getcwd() + ): + pom_path = os.path.join(candidate_path, "pom.xml") + if pom_path in modules_by_path: + resolved.add(modules_by_path[pom_path].key) + break + candidate_path = os.path.dirname(candidate_path) + elif item.endswith("pom.xml") and os.path.abspath(item) in modules_by_path: + resolved.add(modules_by_path[os.path.abspath(item)].key) else: # Try to match simple name (artifactId) or groupId:artifactId found = False for key, module in modules_by_key.items(): - if item == module.artifactId or item == f"{module.group_id}:{module.artifactId}": + if ( + item == module.artifactId + or item == f"{module.group_id}:{module.artifactId}" + ): resolved.add(key) found = True break if not found: - print(f"Warning: Could not resolve input '{item}' to a module.", file=sys.stderr) - + print( + f"Warning: Could not resolve input '{item}' to a module.", + file=sys.stderr, + ) + return list(resolved) + def main(): - parser = argparse.ArgumentParser(description='Identify upstream dependencies for partial builds.') - parser.add_argument('modules', nargs='+', help='List of modified modules or file paths') + parser = argparse.ArgumentParser( + description="Identify upstream dependencies for partial builds." + ) + parser.add_argument( + "modules", nargs="+", help="List of modified modules or file paths" + ) args = parser.parse_args() root_dir = os.getcwd() modules_by_key, graph = build_graph(root_dir) modules_by_path = {m.path: m for m in modules_by_key.values()} - start_nodes = resolve_modules_from_inputs(args.modules, modules_by_path, modules_by_key) - + start_nodes = resolve_modules_from_inputs( + args.modules, modules_by_path, modules_by_key + ) + if not start_nodes: print("No valid modules found from input.", file=sys.stderr) return # Get transitive upstream dependencies - # We include the start nodes themselves in the output set if they are dependencies of other start nodes? + # We include the start nodes themselves in the output set if they are dependencies of other start nodes? # Usually we want: Dependencies of (Start Nodes) NOT INCLUDING Start Nodes themselves, unless A depends on B and both are modified. # But for "installing dependencies", we generally want EVERYTHING upstream of the modified set. # If I modified A, and A depends on B, I want to install B. @@ -240,31 +279,39 @@ def main(): # But usually the build system will build A and B if I say "build A and B". # The request is: "determine which modules will need to be compiled and installed to the local maven repository" # This implies we want the COMPLEMENT set of the modified modules, restricted to the upstream graph. - + all_dependencies = get_transitive_dependencies(start_nodes, graph) - - # Filter out the start nodes themselves, because they are the "modified" ones - # that will presumably be built by the test command itself? - # Actually, usually 'dependencies.sh' installs everything needed for the tests. - # If I am testing A, I need B installed. - # If I am testing A and B, I need C (upstream of B) installed. - # So I need (TransitiveClosure(StartNodes) - StartNodes). - + upstream_only = all_dependencies - set(start_nodes) - - # Map back to paths or artifactIds? - # `mvn -pl` takes directory paths or [groupId]:artifactId - # Directory paths are safer if artifactIds are not unique (rare but possible) - # relpath is good. - + + # Topological sort for installation order + # (Install dependencies before dependents) + sorted_upstream = [] + visited_sort = set() + + def visit(node): + if node in visited_sort: + return + visited_sort.add(node) + # Visit dependencies first + if node in graph: + for dep in graph[node]: + if dep in upstream_only: + visit(dep) + + sorted_upstream.append(node) + + for node in upstream_only: + visit(node) + results = [] - for key in upstream_only: + for key in sorted_upstream: module = modules_by_key[key] rel_path = os.path.relpath(os.path.dirname(module.path), root_dir) - # Maven -pl expects project dir or group:artifact results.append(rel_path) - - print(','.join(sorted(results))) -if __name__ == '__main__': + print(",".join(results)) + + +if __name__ == "__main__": main() From 1cdda3c2716fe731060b28f47e9c80954cef2301 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Wed, 28 Jan 2026 19:47:26 +0000 Subject: [PATCH 3/8] handle dependency management --- .kokoro/determine_dependencies.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.kokoro/determine_dependencies.py b/.kokoro/determine_dependencies.py index 540f80a5d9eb..6629b4af2ae9 100644 --- a/.kokoro/determine_dependencies.py +++ b/.kokoro/determine_dependencies.py @@ -84,15 +84,9 @@ def local_name(tag): module = Module(path, group_id, artifact_id, parent_coords) # Dependencies - def add_dependencies(section, is_management=False): + def add_dependencies(section): if section is not None: for dep in section.findall("mvn:dependency", NS): - if is_management: - # Only include 'import' scope dependencies from dependencyManagement - scope = dep.find("mvn:scope", NS) - if scope is None or scope.text != "import": - continue - d_group = dep.find("mvn:groupId", NS) d_artifact = dep.find("mvn:artifactId", NS) if d_group is not None and d_artifact is not None: @@ -102,7 +96,7 @@ def add_dependencies(section, is_management=False): dep_mgmt = root.find("mvn:dependencyManagement", NS) if dep_mgmt is not None: - add_dependencies(dep_mgmt.find("mvn:dependencies", NS), is_management=True) + add_dependencies(dep_mgmt.find("mvn:dependencies", NS)) # Plugin dependencies build = root.find("mvn:build", NS) @@ -171,6 +165,12 @@ def build_graph( # Regular dependencies for dep_key in module.dependencies: if dep_key in modules: + # Prevent cycle: If I am the parent of the dependency, ignore it. + # (Parent manages Child -> Child depends on Parent) + dep_module = modules[dep_key] + if dep_module.parent == key: + continue + graph[key].add(dep_key) return modules, graph From 856f92cd0402d2f2841bcaa9cb932d577aa770e5 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Wed, 28 Jan 2026 20:16:58 +0000 Subject: [PATCH 4/8] add helper bash function for installing all dependent modules of a single directory --- .kokoro/common.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.kokoro/common.sh b/.kokoro/common.sh index 4f8959c245ef..8d3029fa3811 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -347,3 +347,15 @@ function install_modules() { -T 1C fi } + +scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}")) +root_dir=$(realpath "$scriptDir/..") +function install_dependencies() { + target_module_path=$(realpath "$1") + rel_target_path=$(realpath --relative-to="$root_dir" "$target_module_path") + echo "Resolving dependencies for $rel_target_path..." + dependencies=$(python3 "$scriptDir/determine_dependencies.py" "$1") + echo "Found dependencies: $dependencies" + + install_modules "${dependencies}" +} From 5c5e20573f24b399996f1e9935f610c5470e6c14 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Tue, 3 Feb 2026 22:56:32 +0000 Subject: [PATCH 5/8] use install_dependencies script --- .kokoro/build.sh | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 29d712c681ef..b5bbecfb2698 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -36,15 +36,7 @@ fi if [[ -n "${BUILD_SUBDIR}" ]] then echo "Compiling and building all modules for ${BUILD_SUBDIR}" - mvn clean install \ - -DskipTests \ - -Dclirr.skip \ - -Dflatten.skip \ - -Dcheckstyle.skip \ - -Djacoco.skip \ - -Denforcer.skip \ - --also-make \ - --projects "${BUILD_SUBDIR}" + install_dependencies "${BUILD_SUBDIR}" echo "Running in subdir: ${BUILD_SUBDIR}" pushd "${BUILD_SUBDIR}" fi From 51690fc7015aaa69c8fa3ec5927f49da7b9cf535 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Tue, 3 Feb 2026 23:24:17 +0000 Subject: [PATCH 6/8] install dependencies for dependencies check --- .kokoro/dependencies.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.kokoro/dependencies.sh b/.kokoro/dependencies.sh index 1ea066f5bf77..5805a07b9a18 100755 --- a/.kokoro/dependencies.sh +++ b/.kokoro/dependencies.sh @@ -51,6 +51,8 @@ export MAVEN_OPTS=$(determineMavenOpts) if [[ -n "${BUILD_SUBDIR}" ]] then + echo "Compiling and building all modules for ${BUILD_SUBDIR}" + install_dependencies "${BUILD_SUBDIR}" echo "Running in subdir: ${BUILD_SUBDIR}" pushd "${BUILD_SUBDIR}" fi From 90c9a7a39cf287c69444898e29d16227c712bf0f Mon Sep 17 00:00:00 2001 From: chingor13 Date: Fri, 6 Feb 2026 19:32:37 +0000 Subject: [PATCH 7/8] use install_modules already implemented --- .kokoro/build.sh | 2 +- .kokoro/common.sh | 18 +- .kokoro/determine_dependencies.py | 317 ------------------------------ 3 files changed, 2 insertions(+), 335 deletions(-) delete mode 100644 .kokoro/determine_dependencies.py diff --git a/.kokoro/build.sh b/.kokoro/build.sh index b5bbecfb2698..e2bc20a88836 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -36,7 +36,7 @@ fi if [[ -n "${BUILD_SUBDIR}" ]] then echo "Compiling and building all modules for ${BUILD_SUBDIR}" - install_dependencies "${BUILD_SUBDIR}" + install_modules "${BUILD_SUBDIR}" echo "Running in subdir: ${BUILD_SUBDIR}" pushd "${BUILD_SUBDIR}" fi diff --git a/.kokoro/common.sh b/.kokoro/common.sh index 8d3029fa3811..d27885ea566b 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -56,10 +56,8 @@ function retry_with_backoff { # Given a folder containing a maven multi-module, assign the variable 'submodules' to a # comma-delimited list of /. function parse_submodules() { - pushd "$1" >/dev/null - submodules_array=() - mvn_submodules=$(mvn help:evaluate -Dexpression=project.modules) + mvn_submodules=$(mvn help:evaluate -Dexpression=project.modules -pl "$1") if mvn_submodules=$(grep '<.*>.*' <<< "$mvn_submodules"); then mvn_submodules=$(sed -e 's/<.*>\(.*\)<\/.*>/\1/g' <<< "$mvn_submodules") for submodule in $mvn_submodules; do @@ -77,8 +75,6 @@ function parse_submodules() { echo "${submodules_array[*]}" ) export submodules - - popd >/dev/null } # Given a list of folders containing maven multi-modules, assign the variable 'all_submodules' to a @@ -347,15 +343,3 @@ function install_modules() { -T 1C fi } - -scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}")) -root_dir=$(realpath "$scriptDir/..") -function install_dependencies() { - target_module_path=$(realpath "$1") - rel_target_path=$(realpath --relative-to="$root_dir" "$target_module_path") - echo "Resolving dependencies for $rel_target_path..." - dependencies=$(python3 "$scriptDir/determine_dependencies.py" "$1") - echo "Found dependencies: $dependencies" - - install_modules "${dependencies}" -} diff --git a/.kokoro/determine_dependencies.py b/.kokoro/determine_dependencies.py deleted file mode 100644 index 6629b4af2ae9..000000000000 --- a/.kokoro/determine_dependencies.py +++ /dev/null @@ -1,317 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import os -import sys -import xml.etree.ElementTree as ET -from collections import defaultdict -from typing import Dict, List, Set, Tuple - -# Maven XML namespace -NS = {"mvn": "http://maven.apache.org/POM/4.0.0"} - - -class Module: - def __init__( - self, path: str, group_id: str, artifact_id: str, parent: Tuple[str, str] = None - ): - self.path = path - self.group_id = group_id - self.artifactId = artifact_id - self.parent = parent - self.dependencies: Set[Tuple[str, str]] = set() - - @property - def key(self) -> Tuple[str, str]: - return (self.group_id, self.artifactId) - - def __repr__(self): - return f"{self.group_id}:{self.artifactId}" - - -def parse_pom(path: str) -> Module: - try: - tree = ET.parse(path) - root = tree.getroot() - except ET.ParseError as e: - print(f"Error parsing {path}: {e}", file=sys.stderr) - return None - - # Handle namespace if present - # XML tags in ElementTree are {namespace}tag - # We'll use find with namespaces for robustness, but simple logic for extraction - - # Helper to clean tag name - def local_name(tag): - if "}" in tag: - return tag.split("}", 1)[1] - return tag - - parent_elem = root.find("mvn:parent", NS) - parent_coords = None - parent_group_id = None - if parent_elem is not None: - p_group = parent_elem.find("mvn:groupId", NS).text - p_artifact = parent_elem.find("mvn:artifactId", NS).text - parent_coords = (p_group, p_artifact) - parent_group_id = p_group - - group_id_elem = root.find("mvn:groupId", NS) - # Inherit groupId from parent if not specified - if group_id_elem is not None: - group_id = group_id_elem.text - elif parent_group_id: - group_id = parent_group_id - else: - # Fallback or error? For now, use artifactId as heuristic or empty - group_id = "unknown" - - artifact_id = root.find("mvn:artifactId", NS).text - - module = Module(path, group_id, artifact_id, parent_coords) - - # Dependencies - def add_dependencies(section): - if section is not None: - for dep in section.findall("mvn:dependency", NS): - d_group = dep.find("mvn:groupId", NS) - d_artifact = dep.find("mvn:artifactId", NS) - if d_group is not None and d_artifact is not None: - module.dependencies.add((d_group.text, d_artifact.text)) - - add_dependencies(root.find("mvn:dependencies", NS)) - - dep_mgmt = root.find("mvn:dependencyManagement", NS) - if dep_mgmt is not None: - add_dependencies(dep_mgmt.find("mvn:dependencies", NS)) - - # Plugin dependencies - build = root.find("mvn:build", NS) - if build is not None: - plugins = build.find("mvn:plugins", NS) - if plugins is not None: - for plugin in plugins.findall("mvn:plugin", NS): - # Plugin itself - p_group = plugin.find("mvn:groupId", NS) - p_artifact = plugin.find("mvn:artifactId", NS) - if p_group is not None and p_artifact is not None: - module.dependencies.add((p_group.text, p_artifact.text)) - - # Plugin dependencies - add_dependencies(plugin.find("mvn:dependencies", NS)) - - # Plugin Management - plugin_mgmt = build.find("mvn:pluginManagement", NS) - if plugin_mgmt is not None: - plugins = plugin_mgmt.find("mvn:plugins", NS) - if plugins is not None: - for plugin in plugins.findall("mvn:plugin", NS): - # Plugin itself - p_group = plugin.find("mvn:groupId", NS) - p_artifact = plugin.find("mvn:artifactId", NS) - if p_group is not None and p_artifact is not None: - module.dependencies.add((p_group.text, p_artifact.text)) - - add_dependencies(plugin.find("mvn:dependencies", NS)) - - return module - - -def find_poms(root_dir: str) -> List[str]: - pom_files = [] - for dirpath, dirnames, filenames in os.walk(root_dir): - # Skip hidden directories and known non-module dirs - dirnames[:] = [d for d in dirnames if not d.startswith(".")] - - if "pom.xml" in filenames: - pom_files.append(os.path.join(dirpath, "pom.xml")) - return pom_files - - -def build_graph( - root_dir: str, -) -> Tuple[Dict[Tuple[str, str], Module], Dict[Tuple[str, str], Set[Tuple[str, str]]]]: - pom_paths = find_poms(root_dir) - modules: Dict[Tuple[str, str], Module] = {} - - # First pass: load all modules - for path in pom_paths: - module = parse_pom(path) - if module: - modules[module.key] = module - - # Build adjacency list: dependent -> dependencies (upstream) - # Only include dependencies that are present in the repo - graph: Dict[Tuple[str, str], Set[Tuple[str, str]]] = defaultdict(set) - - for key, module in modules.items(): - # Parent dependency - if module.parent and module.parent in modules: - graph[key].add(module.parent) - - # Regular dependencies - for dep_key in module.dependencies: - if dep_key in modules: - # Prevent cycle: If I am the parent of the dependency, ignore it. - # (Parent manages Child -> Child depends on Parent) - dep_module = modules[dep_key] - if dep_module.parent == key: - continue - - graph[key].add(dep_key) - - return modules, graph - - -def get_transitive_dependencies( - start_nodes: List[Tuple[str, str]], - graph: Dict[Tuple[str, str], Set[Tuple[str, str]]], -) -> Set[Tuple[str, str]]: - visited = set() - stack = list(start_nodes) - - while stack: - node = stack.pop() - if node not in visited: - visited.add(node) - # Add upstream dependencies to stack - if node in graph: - for upstream in graph[node]: - if upstream not in visited: - stack.append(upstream) - - return visited - - -def resolve_modules_from_inputs( - inputs: List[str], - modules_by_path: Dict[str, Module], - modules_by_key: Dict[Tuple[str, str], Module], -) -> List[Tuple[str, str]]: - resolved = set() - for item in inputs: - # Check if item is a path - abs_item = os.path.abspath(item) - - # If it's a file, try to find the nearest pom.xml - if os.path.isfile(abs_item) or ( - not item.endswith("pom.xml") and os.path.isdir(abs_item) - ): - # Heuristic: if it's a file, find containing pom - # if it's a dir, look for pom.xml inside or check if it matches a module path - candidate_path = abs_item - if os.path.isfile(candidate_path) and not candidate_path.endswith( - "pom.xml" - ): - candidate_path = os.path.dirname(candidate_path) - - # Traverse up to find pom.xml - while candidate_path.startswith(os.getcwd()) and len(candidate_path) >= len( - os.getcwd() - ): - pom_path = os.path.join(candidate_path, "pom.xml") - if pom_path in modules_by_path: - resolved.add(modules_by_path[pom_path].key) - break - candidate_path = os.path.dirname(candidate_path) - elif item.endswith("pom.xml") and os.path.abspath(item) in modules_by_path: - resolved.add(modules_by_path[os.path.abspath(item)].key) - else: - # Try to match simple name (artifactId) or groupId:artifactId - found = False - for key, module in modules_by_key.items(): - if ( - item == module.artifactId - or item == f"{module.group_id}:{module.artifactId}" - ): - resolved.add(key) - found = True - break - if not found: - print( - f"Warning: Could not resolve input '{item}' to a module.", - file=sys.stderr, - ) - - return list(resolved) - - -def main(): - parser = argparse.ArgumentParser( - description="Identify upstream dependencies for partial builds." - ) - parser.add_argument( - "modules", nargs="+", help="List of modified modules or file paths" - ) - args = parser.parse_args() - - root_dir = os.getcwd() - modules_by_key, graph = build_graph(root_dir) - modules_by_path = {m.path: m for m in modules_by_key.values()} - - start_nodes = resolve_modules_from_inputs( - args.modules, modules_by_path, modules_by_key - ) - - if not start_nodes: - print("No valid modules found from input.", file=sys.stderr) - return - - # Get transitive upstream dependencies - # We include the start nodes themselves in the output set if they are dependencies of other start nodes? - # Usually we want: Dependencies of (Start Nodes) NOT INCLUDING Start Nodes themselves, unless A depends on B and both are modified. - # But for "installing dependencies", we generally want EVERYTHING upstream of the modified set. - # If I modified A, and A depends on B, I want to install B. - # If I modified A and B, and A depends on B, I want to install B (before A). - # But usually the build system will build A and B if I say "build A and B". - # The request is: "determine which modules will need to be compiled and installed to the local maven repository" - # This implies we want the COMPLEMENT set of the modified modules, restricted to the upstream graph. - - all_dependencies = get_transitive_dependencies(start_nodes, graph) - - upstream_only = all_dependencies - set(start_nodes) - - # Topological sort for installation order - # (Install dependencies before dependents) - sorted_upstream = [] - visited_sort = set() - - def visit(node): - if node in visited_sort: - return - visited_sort.add(node) - # Visit dependencies first - if node in graph: - for dep in graph[node]: - if dep in upstream_only: - visit(dep) - - sorted_upstream.append(node) - - for node in upstream_only: - visit(node) - - results = [] - for key in sorted_upstream: - module = modules_by_key[key] - rel_path = os.path.relpath(os.path.dirname(module.path), root_dir) - results.append(rel_path) - - print(",".join(results)) - - -if __name__ == "__main__": - main() From 4a2167584b6225ab568362dadfd57f5181d405c9 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Sat, 7 Feb 2026 00:01:06 +0000 Subject: [PATCH 8/8] add logging split tests build configs --- .kokoro/build.sh | 39 +++++++++++++++++++ .kokoro/common.sh | 8 +++- .../logging-graalvm-native-presubmit.cfg | 38 ++++++++++++++++++ .kokoro/presubmit/logging-integration.cfg | 38 ++++++++++++++++++ 4 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 .kokoro/presubmit/logging-graalvm-native-presubmit.cfg create mode 100644 .kokoro/presubmit/logging-integration.cfg diff --git a/.kokoro/build.sh b/.kokoro/build.sh index e2bc20a88836..7b625e4ebead 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -77,6 +77,28 @@ case ${JOB_TYPE} in echo "No Integration Tests to run" fi ;; + integration-single) + echo "SUREFIRE_JVM_OPT: ${SUREFIRE_JVM_OPT}" + echo "INTEGRATION_TEST_ARGS: ${INTEGRATION_TEST_ARGS}" + mvn verify -Penable-integration-tests \ + ${INTEGRATION_TEST_ARGS} \ + -B -ntp -fae \ + -DtrimStackTrace=false \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -Dorg.slf4j.simpleLogger.showDateTime=true \ + -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \ + -Dcheckstyle.skip=true \ + -Dflatten.skip=true \ + -Danimal.sniffer.skip=true \ + -Djacoco.skip=true \ + -DskipUnitTests=true \ + -Dmaven.wagon.http.retryHandler.count=5 \ + -T 1C ${SUREFIRE_JVM_OPT} + + RETURN_CODE=$? + printf "Finished integration tests for modules:\n%s\n" "${BUILD_SUBDIR}" + ;; graalvm-presubmit) generate_graalvm_presubmit_modules_list if [[ "$(release_please_snapshot_pull_request)" == "true" ]]; then @@ -101,6 +123,23 @@ case ${JOB_TYPE} in echo "Not running GraalVM checks -- No changes in relevant modules" fi ;; + graalvm-single) + echo "INTEGRATION_TEST_ARGS: ${INTEGRATION_TEST_ARGS}" + mvn test -Pnative \ + ${INTEGRATION_TEST_ARGS} \ + -B -ntp -fae \ + -DtrimStackTrace=false \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -Dorg.slf4j.simpleLogger.showDateTime=true \ + -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \ + -Dcheckstyle.skip=true \ + -Dflatten.skip=true \ + -Danimal.sniffer.skip=true + + RETURN_CODE=$? + printf "Finished GraalVM ITs for modules:\n%s\n" "${BUILD_SUBDIR}" + ;; lint) if [ -n "${BASE_SHA}" ] && [ -n "${HEAD_SHA}" ]; then changed_file_list=$(git diff --name-only "${BASE_SHA}" "${HEAD_SHA}") diff --git a/.kokoro/common.sh b/.kokoro/common.sh index d27885ea566b..c33a23c5bbe4 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -13,7 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -excluded_modules=('gapic-libraries-bom' 'google-cloud-jar-parent' 'google-cloud-pom-parent' 'java-vertexai') +excluded_modules=( + 'gapic-libraries-bom' + 'google-cloud-jar-parent' + 'google-cloud-pom-parent' + 'java-vertexai', + 'java-logging' +) function retry_with_backoff { attempts_left=$1 diff --git a/.kokoro/presubmit/logging-graalvm-native-presubmit.cfg b/.kokoro/presubmit/logging-graalvm-native-presubmit.cfg new file mode 100644 index 000000000000..d16598bb817f --- /dev/null +++ b/.kokoro/presubmit/logging-graalvm-native-presubmit.cfg @@ -0,0 +1,38 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.56.1" +} + +env_vars: { + key: "JOB_TYPE" + value: "graalvm-single" +} + +# TODO: remove this after we've migrated all tests and scripts +env_vars: { + key: "GCLOUD_PROJECT" + value: "cloud-java-ci-test" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "cloud-java-ci-test" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/cloud-java-ci-it-service-account" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "cloud-java-ci-it-service-account, java-bigqueryconnection-samples-secrets" +} + +env_vars: { + key: "BUILD_SUBDIR" + value: "java-logging" +} diff --git a/.kokoro/presubmit/logging-integration.cfg b/.kokoro/presubmit/logging-integration.cfg new file mode 100644 index 000000000000..30143885f2b4 --- /dev/null +++ b/.kokoro/presubmit/logging-integration.cfg @@ -0,0 +1,38 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/java11" +} + +env_vars: { + key: "JOB_TYPE" + value: "integration-single" +} + +# TODO: remove this after we've migrated all tests and scripts +env_vars: { + key: "GCLOUD_PROJECT" + value: "cloud-java-ci-test" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "cloud-java-ci-test" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/cloud-java-ci-it-service-account" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "cloud-java-ci-it-service-account, java-bigqueryconnection-samples-secrets" +} + +env_vars: { + key: "BUILD_SUBDIR" + value: "java-logging" +}