feat: enable pyproject.toml as single source of truth for Python version

.gitattributes

@@ -1,2 +1,3 @@
 python/features.bzl export-subst
 tools/publish/*.txt linguist-generated=true
+requirements_lock.txt linguist-generated=true

.gitignore

@@ -48,8 +48,10 @@ user.bazelrc
 # CLion
 .clwb
 
-# Python cache
+# Python artifacts
 **/__pycache__/
+*.egg
+*.egg-info
 
 # MODULE.bazel.lock is ignored for now as per recommendation from upstream.
 # See https://github.com/bazelbuild/bazel/issues/20369

CHANGELOG.md

@@ -74,6 +74,8 @@ END_UNRELEASED_TEMPLATE
 
 {#v0-0-0-added}
 ### Added
+* (pip,python) Added `pyproject_toml` attribute to `pip.default()` and `python.defaults()`
+  to read Python version from pyproject.toml `requires-python` field (must be `==X.Y.Z` format).
 * (toolchain) Added {obj}`python.override.toolchain_target_settings` to allow
   adding `config_setting` labels to all registered toolchains.
 * (windows) Full venv support for Windows is available. Set

python/private/BUILD.bazel

@@ -274,6 +274,8 @@ bzl_library(
     deps = [
         ":full_version_bzl",
         ":platform_info_bzl",
+        ":pyproject_repo_bzl",
+        ":pyproject_utils_bzl",
         ":python_register_toolchains_bzl",
         ":pythons_hub_bzl",
         ":repo_utils_bzl",
@@ -751,6 +753,16 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "pyproject_repo_bzl",
+    srcs = ["pyproject_repo.bzl"],
+)
+
+bzl_library(
+    name = "pyproject_utils_bzl",
+    srcs = ["pyproject_utils.bzl"],
+)
+
 # Needed to define bzl_library targets for docgen. (We don't define the
 # bzl_library target here because it'd give our users a transitive dependency
 # on Skylib.)

python/private/pypi/BUILD.bazel

@@ -131,6 +131,7 @@ bzl_library(
         ":whl_library_bzl",
         "//python/private:auth_bzl",
         "//python/private:normalize_name_bzl",
+        "//python/private:pyproject_utils_bzl",
         "//python/private:repo_utils_bzl",
         "@bazel_features//:features",
         "@pythons_hub//:interpreters_bzl",

python/private/pypi/extension.bzl

@@ -19,6 +19,7 @@ load("@pythons_hub//:versions.bzl", "MINOR_MAPPING")
 load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config")
 load("//python/private:auth.bzl", "AUTH_ATTRS")
 load("//python/private:normalize_name.bzl", "normalize_name")
+load("//python/private:pyproject_utils.bzl", "read_pyproject_version")
 load("//python/private:repo_utils.bzl", "repo_utils")
 load(":hub_builder.bzl", "hub_builder")
 load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json")
@@ -205,12 +206,23 @@ def build_config(
     """
     defaults = {
         "platforms": default_platforms(),
+        "python_version": None,
     }
     for mod in module_ctx.modules:
         if not (mod.is_root or mod.name == "rules_python"):
             continue
 
         for tag in mod.tags.default:
+            pyproject_toml = tag.pyproject_toml
+            if pyproject_toml:
+                pyproject_version = read_pyproject_version(
+                    module_ctx,
+                    pyproject_toml,
+                    logger = None,
+                )
+                if pyproject_version:
+                    defaults["python_version"] = pyproject_version
+
             platform = tag.platform
             if platform:
                 specific_config = defaults["platforms"].setdefault(platform, {})
@@ -246,6 +258,7 @@ def build_config(
         auth_patterns = defaults.get("auth_patterns", {}),
         index_url = defaults.get("index_url", "https://pypi.org/simple").rstrip("/"),
         netrc = defaults.get("netrc", None),
+        python_version = defaults.get("python_version", None),
         platforms = {
             name: _plat(**values)
             for name, values in defaults["platforms"].items()
@@ -345,6 +358,10 @@ You cannot use both the additive_build_content and additive_build_content_file a
 
     for mod in module_ctx.modules:
         for pip_attr in mod.tags.parse:
+            python_version = pip_attr.python_version or config.python_version
+            if not python_version:
+                _fail("pip.parse() requires either python_version attribute or pip.default(pyproject_toml=...) to be set")
+
             hub_name = pip_attr.hub_name
             if hub_name not in pip_hub_map:
                 builder = hub_builder(
@@ -381,6 +398,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
             builder.pip_parse(
                 module_ctx,
                 pip_attr = pip_attr,
+                python_version = python_version,
             )
 
     # Keeps track of all the hub's whl repos across the different versions.
@@ -536,7 +554,7 @@ Either this or {attr}`env` `platform_machine` key should be specified.
 """,
     ),
     "config_settings": attr.label_list(
-        mandatory = True,
+        mandatory = False,
         doc = """\
 The list of labels to `config_setting` targets that need to be matched for the platform to be
 selected.
@@ -618,6 +636,21 @@ If you are defining custom platforms in your project and don't want things to cl
 [isolation] feature.
 
 [isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate
+""",
+    ),
+    "pyproject_toml": attr.label(
+        mandatory = False,
+        doc = """\
+Label pointing to pyproject.toml file to read the default Python version from.
+When specified, reads the `requires-python` field from pyproject.toml and uses
+it as the default python_version for all `pip.parse()` calls that don't
+explicitly specify one.
+
+The version must be specified as `==X.Y.Z` (exact version with full semver).
+This is designed to work with dependency management tools like Renovate.
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
 """,
     ),
     "whl_abi_tags": attr.string_list(
@@ -778,14 +811,18 @@ find in case extra indexes are specified.
             default = True,
         ),
         "python_version": attr.string(
-            mandatory = True,
+            mandatory = False,
             doc = """
 The Python version the dependencies are targetting, in Major.Minor format
 (e.g., "3.11") or patch level granularity (e.g. "3.11.1").
 
 If an interpreter isn't explicitly provided (using `python_interpreter` or
 `python_interpreter_target`), then the version specified here must have
 a corresponding `python.toolchain()` configured.
+
+:::{seealso}
+The {obj}`pyproject_toml` attribute for getting the version from a project file.
+:::
 """,
         ),
         "simpleapi_skip": attr.string_list(

python/private/pypi/hub_builder.bzl

@@ -147,8 +147,8 @@ def _build(self):
         whl_libraries = self._whl_libraries,
     )
 
-def _pip_parse(self, module_ctx, pip_attr):
-    python_version = pip_attr.python_version
+def _pip_parse(self, module_ctx, pip_attr, python_version = None):
+    python_version = python_version or pip_attr.python_version
     if python_version in self._platforms:
         fail((
             "Duplicate pip python version '{version}' for hub " +
@@ -194,7 +194,8 @@ def _pip_parse(self, module_ctx, pip_attr):
         self,
         module_ctx,
         pip_attr = pip_attr,
-        enable_pipstar_extract = bool(self._config.enable_pipstar_extract or self._get_index_urls.get(pip_attr.python_version)),
+        python_version = python_version,
+        enable_pipstar_extract = bool(self._config.enable_pipstar_extract or self._get_index_urls.get(python_version)),
     )
 
 ### end of PUBLIC methods
@@ -387,11 +388,11 @@ def _set_get_index_urls(self, pip_attr):
     )
     return True
 
-def _detect_interpreter(self, pip_attr):
+def _detect_interpreter(self, pip_attr, python_version):
     python_interpreter_target = pip_attr.python_interpreter_target
     if python_interpreter_target == None and not pip_attr.python_interpreter:
         python_name = "python_{}_host".format(
-            pip_attr.python_version.replace(".", "_"),
+            python_version.replace(".", "_"),
         )
         if python_name not in self._available_interpreters:
             fail((
@@ -401,7 +402,7 @@ def _detect_interpreter(self, pip_attr):
                 "Expected to find {python_name} among registered versions:\n  {labels}"
             ).format(
                 hub_name = self.name,
-                version = pip_attr.python_version,
+                version = python_version,
                 python_name = python_name,
                 labels = "  \n".join(self._available_interpreters),
             ))
@@ -465,31 +466,33 @@ def _platforms(module_ctx, *, python_version, config, target_platforms):
         )
     return platforms
 
-def _evaluate_markers(self, pip_attr):
+def _evaluate_markers(self, python_version):
     if self._evaluate_markers_fn:
         return self._evaluate_markers_fn
 
     return lambda _, requirements: evaluate_markers_star(
         requirements = requirements,
-        platforms = self._platforms[pip_attr.python_version],
+        platforms = self._platforms[python_version],
     )
 
 def _create_whl_repos(
         self,
         module_ctx,
         *,
         pip_attr,
+        python_version,
         enable_pipstar_extract = False):
     """create all of the whl repositories
 
     Args:
         self: the builder.
         module_ctx: {type}`module_ctx`.
         pip_attr: {type}`struct` - the struct that comes from the tag class iteration.
+        python_version: {type}`str` - the resolved python version for this pip.parse call.
         enable_pipstar_extract: {type}`bool` - enable the pipstar extraction or not.
     """
     logger = self._logger
-    platforms = self._platforms[pip_attr.python_version]
+    platforms = self._platforms[python_version]
     requirements_by_platform = parse_requirements(
         module_ctx,
         requirements_by_platform = requirements_files_by_platform(
@@ -501,15 +504,15 @@ def _create_whl_repos(
             extra_pip_args = pip_attr.extra_pip_args,
             platforms = sorted(platforms),  # here we only need keys
             python_version = full_version(
-                version = pip_attr.python_version,
+                version = python_version,
                 minor_mapping = self._minor_mapping,
             ),
             logger = logger,
         ),
         platforms = platforms,
         extra_pip_args = pip_attr.extra_pip_args,
-        get_index_urls = self._get_index_urls.get(pip_attr.python_version),
-        evaluate_markers = _evaluate_markers(self, pip_attr),
+        get_index_urls = self._get_index_urls.get(python_version),
+        evaluate_markers = _evaluate_markers(self, python_version),
         logger = logger,
     )
 
@@ -530,7 +533,7 @@ def _create_whl_repos(
         pip_attr = pip_attr,
     )
 
-    interpreter = _detect_interpreter(self, pip_attr)
+    interpreter = _detect_interpreter(self, pip_attr, python_version)
 
     for whl in requirements_by_platform:
         whl_library_args = common_args | _whl_library_args(
@@ -545,16 +548,16 @@ def _create_whl_repos(
                 whl_library_args = whl_library_args,
                 download_only = pip_attr.download_only,
                 netrc = self._config.netrc or pip_attr.netrc,
-                use_downloader = src.url and _use_downloader(self, pip_attr.python_version, whl.name),
+                use_downloader = src.url and _use_downloader(self, python_version, whl.name),
                 auth_patterns = self._config.auth_patterns or pip_attr.auth_patterns,
-                python_version = _major_minor_version(pip_attr.python_version),
+                python_version = _major_minor_version(python_version),
                 is_multiple_versions = whl.is_multiple_versions,
                 interpreter = interpreter,
                 enable_pipstar_extract = enable_pipstar_extract,
             )
             _add_whl_library(
                 self,
-                python_version = pip_attr.python_version,
+                python_version = python_version,
                 whl = whl,
                 repo = repo,
             )

python/private/pyproject_repo.bzl

@@ -0,0 +1,80 @@
+"""Repository rule to expose Python version from pyproject.toml."""
+
+_TOML2JSON = Label("//tools/private/toml2json:toml2json.py")
+
+def _parse_requires_python(requires_python):
+    """Parse and validate the requires-python field."""
+    if not requires_python.startswith("=="):
+        fail("requires-python must use '==' for exact version, got: {}".format(requires_python))
+
+    bare_version = requires_python[2:].strip()
+    parts = bare_version.split(".")
+    if len(parts) != 3:
+        fail("requires-python must be in X.Y.Z format, got: {}".format(bare_version))
+    for part in parts:
+        if not part.isdigit():
+            fail("requires-python must be in X.Y.Z format, got: {}".format(bare_version))
+
+    return bare_version
+
+def _pyproject_version_repo_impl(rctx):
+    """Create a repository that exports PYTHON_VERSION from pyproject.toml."""
+    pyproject_path = rctx.path(rctx.attr.pyproject_toml)
+    rctx.read(pyproject_path, watch = "yes")
+
+    toml2json = rctx.path(_TOML2JSON)
+    result = rctx.execute([
+        "python3",
+        str(toml2json),
+        str(pyproject_path),
+    ])
+
+    if result.return_code != 0:
+        fail("Failed to parse pyproject.toml: " + result.stderr)
+
+    data = json.decode(result.stdout)
+    requires_python = data.get("project", {}).get("requires-python")
+    if not requires_python:
+        fail("pyproject.toml must contain [project] requires-python field")
+
+    version = _parse_requires_python(requires_python)
+
+    rctx.file("version.bzl", """\
+\"\"\"Python version from pyproject.toml.
+
+This file is automatically generated. Do not edit.
+\"\"\"
+
+PYTHON_VERSION = "{version}"
+""".format(version = version))
+
+    rctx.file("BUILD.bazel", """\
+# Automatically generated from pyproject.toml
+exports_files(["version.bzl"])
+""")
+
+pyproject_version_repo = repository_rule(
+    implementation = _pyproject_version_repo_impl,
+    attrs = {
+        "pyproject_toml": attr.label(
+            mandatory = True,
+            doc = "Label pointing to pyproject.toml file.",
+        ),
+    },
+    doc = """Repository rule that reads Python version from pyproject.toml.
+
+This rule creates a repository with a `version.bzl` file that exports
+`PYTHON_VERSION` constant.
+
+Example:
+```python
+    load("@python_version_from_pyproject//:version.bzl", "PYTHON_VERSION")
+
+    compile_pip_requirements(
+        name = "requirements",
+        python_version = PYTHON_VERSION,
+        requirements_txt = "requirements.txt",
+    )
+```
+""",
+)