From f724c034049bd3f31b79359f0f9809cbe1b0828b Mon Sep 17 00:00:00 2001 From: rgaudin Date: Thu, 5 Mar 2026 14:08:44 +0000 Subject: [PATCH 1/4] Stack update - Drop python 3.8 and 3.9 - update dependencies and actions - use ruff instead of black --- .github/workflows/Publish.yaml | 8 ++-- .github/workflows/QA.yaml | 9 ++-- .github/workflows/Tests.yaml | 14 +++--- .pre-commit-config.yaml | 21 --------- pyproject.toml | 54 ++++++++++------------ src/docker_export/__init__.py | 6 +-- tasks.py | 84 +++++++++++++++------------------- 7 files changed, 79 insertions(+), 117 deletions(-) delete mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/Publish.yaml b/.github/workflows/Publish.yaml index a438147..e6df6cf 100644 --- a/.github/workflows/Publish.yaml +++ b/.github/workflows/Publish.yaml @@ -6,15 +6,15 @@ on: jobs: publish: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: id-token: write # mandatory for PyPI trusted publishing steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: python-version-file: pyproject.toml architecture: x64 @@ -25,7 +25,7 @@ jobs: python -m build --sdist --wheel - name: Upload to PyPI - uses: pypa/gh-action-pypi-publish@release/v1.8 + uses: pypa/gh-action-pypi-publish@release/v1 # dont specify anything for Trusted Publishing # https://docs.pypi.org/trusted-publishers # with: diff --git a/.github/workflows/QA.yaml b/.github/workflows/QA.yaml index 3053e04..7f8509a 100644 --- a/.github/workflows/QA.yaml +++ b/.github/workflows/QA.yaml @@ -8,13 +8,13 @@ on: jobs: check-qa: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: python-version-file: pyproject.toml architecture: x64 @@ -24,9 +24,6 @@ jobs: pip install -U pip pip install -e .[lint,scripts,test,check,visual] - - name: Check black formatting - run: inv lint-black - - name: Check ruff run: inv lint-ruff diff --git a/.github/workflows/Tests.yaml b/.github/workflows/Tests.yaml index 919be49..2cbf3e6 100644 --- a/.github/workflows/Tests.yaml +++ b/.github/workflows/Tests.yaml @@ -10,15 +10,15 @@ jobs: run-tests: strategy: matrix: - os: [ubuntu-22.04] - python: ["3.8", "3.9", "3.10", "3.11"] + os: [ubuntu-24.04] + python: ["3.10", "3.11", "3.12", "3.13", "3.14"] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python }} architecture: x64 @@ -32,18 +32,18 @@ jobs: run: inv coverage --args "-vvv" - name: Upload coverage report to codecov - if: matrix.python == '3.11' + if: matrix.python == '3.10' uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} build_python: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: python-version-file: pyproject.toml architecture: x64 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 04d7cb2..0000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# See https://pre-commit.com for more information -# See https://pre-commit.com/hooks.html for more hooks -repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 - hooks: - - id: trailing-whitespace - - id: end-of-file-fixer -- repo: https://github.com/psf/black - rev: "24.1.1" - hooks: - - id: black -- repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.15 - hooks: - - id: ruff -- repo: https://github.com/RobertCraigie/pyright-python - rev: v1.1.349 - hooks: - - id: pyright - entry: hatch run check:all diff --git a/pyproject.toml b/pyproject.toml index e1a2094..f57cb6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,42 +8,44 @@ authors = [ { name = "Kiwix", email = "dev@kiwix.org" }, ] keywords = ["oci", "image", "docker", "kiwix"] -requires-python = ">=3.8" +requires-python = ">=3.10" description = "Export docker image into tar file directly from registry API" readme = "README.md" license = {text = "GPL-3.0-or-later"} classifiers = [ "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", ] dependencies = [ "requests>=2,<3", - "pathvalidate==3.2.0", + "types-requests>=2,<3", + "pathvalidate>=3.2.0", ] dynamic = ["version"] [project.optional-dependencies] scripts = [ - "invoke==2.2.0", + "invoke==2.2.1", ] lint = [ - "black==24.1.1", - "ruff==0.1.15", + "black==26.1.0", + "ruff==0.15.4", ] check = [ - "pyright==1.1.349", + "pyright==1.1.408", ] test = [ - "pytest==8.0.0", - "coverage==7.4.1", + "pytest==9.0.2", + "coverage==7.13.4", ] dev = [ - "pre-commit==3.6.0", - "debugpy==1.6.7", + "pre-commit==4.5.1", + "debugpy==1.8.20", "docker-export[scripts]", "docker-export[lint]", "docker-export[test]", @@ -51,7 +53,8 @@ dev = [ "docker-export[visual]", ] visual = [ - "humanfriendly>=8.0", + "humanfriendly>=10.0", + "types-humanfriendly>=10", "progressbar2>=4.0" ] @@ -71,7 +74,7 @@ exclude = [ ] [[tool.hatch.envs.default.matrix]] -python = ["3.8"] +python = ["3.10"] [tool.hatch.envs.default] features = ["dev"] @@ -80,7 +83,7 @@ features = ["dev"] features = ["scripts", "test", "visual"] [[tool.hatch.envs.test.matrix]] -python = ["3.8", "3.9", "3.10", "3.11"] +python = ["3.10", "3.11", "3.12", "3.13", "3.14"] [tool.hatch.envs.test.scripts] run = "inv test --args '{args}'" @@ -94,10 +97,8 @@ skip-install = false features = ["scripts", "lint", "visual"] [tool.hatch.envs.lint.scripts] -black = "inv lint-black --args '{args}'" ruff = "inv lint-ruff --args '{args}'" all = "inv lintall --args '{args}'" -fix-black = "inv fix-black --args '{args}'" fix-ruff = "inv fix-ruff --args '{args}'" fixall = "inv fixall --args '{args}'" @@ -108,14 +109,13 @@ features = ["scripts", "check", "test", "visual"] pyright = "inv check-pyright --args '{args}'" all = "inv checkall --args '{args}'" -[tool.black] -line-length = 88 -target-version = ['py38'] [tool.ruff] -target-version = "py38" +target-version = "py310" line-length = 88 src = ["src"] + +[tool.ruff.lint] select = [ "A", # flake8-builtins # "ANN", # flake8-annotations @@ -194,17 +194,13 @@ unfixable = [ "F401", ] -[tool.ruff.isort] +[tool.ruff.lint.isort] known-first-party = ["docker_export"] -[tool.ruff.flake8-bugbear] -# add exceptions to B008 for fastapi. -extend-immutable-calls = ["fastapi.Depends", "fastapi.Query"] - -[tool.ruff.flake8-tidy-imports] +[tool.ruff.lint.flake8-tidy-imports] ban-relative-imports = "all" -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] # Tests can use magic values, assertions, and relative imports "tests/**/*" = ["PLR2004", "S101", "TID252"] @@ -236,6 +232,6 @@ exclude_lines = [ include = ["src", "tests", "tasks.py"] exclude = [".env/**", ".venv/**"] extraPaths = ["src"] -pythonVersion = "3.8" +pythonVersion = "3.10" typeCheckingMode="strict" reportImplicitStringConcatenation=false diff --git a/src/docker_export/__init__.py b/src/docker_export/__init__.py index 072e23a..3c00284 100755 --- a/src/docker_export/__init__.py +++ b/src/docker_export/__init__.py @@ -27,7 +27,7 @@ except ImportError: progressbar = None try: - import humanfriendly + import humanfriendly # pyright: ignore [reportMissingTypeStubs] except ImportError: humanfriendly = None @@ -128,8 +128,8 @@ def finish(self): print("") -@dataclass -class Platform: +@dataclass(frozen=True) +class Platform: # noqa: PLW1641 architecture: str os: str variant: str diff --git a/tasks.py b/tasks.py index 6fb0d18..0a5ad5e 100644 --- a/tasks.py +++ b/tasks.py @@ -1,6 +1,4 @@ # pyright: strict, reportUntypedFunctionDecorator=false -from __future__ import annotations - import os from invoke.context import Context @@ -10,95 +8,87 @@ @task(optional=["args"], help={"args": "pytest additional arguments"}) -def test(ctx: Context, args: str | None = ""): +def test(ctx: Context, args: str = ""): """run tests (without coverage)""" ctx.run(f"pytest {args}", pty=use_pty) @task(optional=["args"], help={"args": "pytest additional arguments"}) -def test_cov(ctx: Context, args: str | None = ""): +def test_cov(ctx: Context, args: str = ""): """run test vith coverage""" ctx.run(f"coverage run -m pytest {args}", pty=use_pty) -@task() -def report_cov(ctx: Context): +@task(optional=["html"], help={"html": "flag to export html report"}) +def report_cov(ctx: Context, *, html: bool = False): """report coverage""" ctx.run("coverage combine", warn=True, pty=use_pty) ctx.run("coverage report --show-missing", pty=use_pty) - - -@task(optional=["args"], help={"args": "pytest additional arguments"}) -def coverage(ctx: Context, args: str | None = ""): - """run tests and report coverage""" - test_cov(ctx, args) - report_cov(ctx) + ctx.run("coverage xml", pty=use_pty) + if html: + ctx.run("coverage html", pty=use_pty) @task( - optional=["args"], help={"args": "linting tools (black, ruff) additional arguments"} + optional=["args", "html"], + help={ + "args": "pytest additional arguments", + "html": "flag to export html report", + }, ) -def lint_black(ctx: Context, args: str | None = ""): - args = args or "." - ctx.run("black --version", pty=use_pty) - ctx.run(f"black --check --diff {args}", pty=use_pty) +def coverage(ctx: Context, args: str = "", *, html: bool = False): + """run tests and report coverage""" + test_cov(ctx, args=args) + report_cov(ctx, html=html) -@task( - optional=["args"], help={"args": "linting tools (black, ruff) additional arguments"} -) -def lint_ruff(ctx: Context, args: str | None = ""): - args = args or "." +@task(optional=["args"], help={"args": "ruff additional arguments"}) +def lint_ruff(ctx: Context, args: str = "."): + args = args or "." # needed for hatch script ctx.run("ruff --version", pty=use_pty) ctx.run(f"ruff check {args}", pty=use_pty) @task( - optional=["args"], help={"args": "linting tools (black, ruff) additional arguments"} + optional=["args"], + help={ + "args": "linting tools (ruff) additional arguments, typically a path", + }, ) -def lintall(ctx: Context, args: str | None = ""): - """check linting""" - args = args or "." - lint_black(ctx, args) +def lintall(ctx: Context, args: str = "."): + """Check linting""" + args = args or "." # needed for hatch script lint_ruff(ctx, args) @task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) -def check_pyright(ctx: Context, args: str | None = ""): +def check_pyright(ctx: Context, args: str = ""): """check static types with pyright""" - args = args or "" ctx.run("pyright --version") ctx.run(f"pyright {args}", pty=use_pty) @task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) -def checkall(ctx: Context, args: str | None = ""): +def checkall(ctx: Context, args: str = ""): """check static types""" - args = args or "" check_pyright(ctx, args) -@task(optional=["args"], help={"args": "black additional arguments"}) -def fix_black(ctx: Context, args: str | None = ""): - """fix black formatting""" - args = args or "." - ctx.run(f"black {args}", pty=use_pty) - - @task(optional=["args"], help={"args": "ruff additional arguments"}) -def fix_ruff(ctx: Context, args: str | None = ""): +def fix_ruff(ctx: Context, args: str = "."): """fix all ruff rules""" - args = args or "." - ctx.run(f"ruff --fix {args}", pty=use_pty) + args = args or "." # needed for hatch script + ctx.run(f"ruff check --fix {args}", pty=use_pty) @task( optional=["args"], - help={"args": "linting (fix mode) tools (black, ruff) additional arguments"}, + help={ + "args": "linting tools (ruff) additional arguments, typically a path", + }, ) -def fixall(ctx: Context, args: str | None = ""): - """fix everything automatically""" - args = args or "." - fix_black(ctx, args) +def fixall(ctx: Context, args: str = "."): + """Fix everything automatically""" + args = args or "." # needed for hatch script fix_ruff(ctx, args) lintall(ctx, args) From 355f281e452602ab5cc49db4d2fd6ff2312fe162 Mon Sep 17 00:00:00 2001 From: rgaudin Date: Thu, 5 Mar 2026 16:41:10 +0000 Subject: [PATCH 2/4] Fixed get_digest test That particular test was returning an unexpected digest at time the test was written. A comment attest this. It is still returning a different value than the one shown in docker hub. --- tests/test_digest.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_digest.py b/tests/test_digest.py index 7fa07c1..a36f3b3 100644 --- a/tests/test_digest.py +++ b/tests/test_digest.py @@ -19,9 +19,10 @@ ( "helloysd/caddy:0.10.11", "linux/amd64", - "672342e30b241ae37f442b0afdd6e744f776af3bd602c4bceaa8dcf2547841bd", - # different from hub (v1 manifest) - # "sha256:8a7b91584f5d0ee6211249d05ec51f026b763d3e1a87885e7d0d6968c42ad6b1", + # on docker hub digest is shown as + # "sha256:8a7b91584f5d0ee6211249d05ec51f026b763d3e1a87885e7d0d6968c42ad6b1" + # which is the Docker-Content-Digest + "sha256:148e55d6f2c3fa74bdba8f2b6870677cfcb268e5a9bebc9e5135a026f502f447", ), ( f"{kiwix_tools}:3.5.0-2", From f9538520d3c32487e7eccb346bbf40b9fac49582 Mon Sep 17 00:00:00 2001 From: rgaudin Date: Thu, 5 Mar 2026 16:41:39 +0000 Subject: [PATCH 3/4] Fixed double slash in URL --- src/docker_export/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docker_export/__init__.py b/src/docker_export/__init__.py index 3c00284..817418a 100755 --- a/src/docker_export/__init__.py +++ b/src/docker_export/__init__.py @@ -260,7 +260,7 @@ def url(self) -> str: "hub.docker.com" if self.registry == "index.docker.io" else self.registry ) prefix = "r/" if self.registry == "index.docker.io" else "" - return f"https://{domain}/{prefix}/{self.fullname}" + return f"https://{domain}/{prefix}{'/' if prefix else ''}{self.fullname}" @classmethod def parse( From f9145098bdde408a2a0e7630ceb6719368ea071a Mon Sep 17 00:00:00 2001 From: rgaudin Date: Thu, 5 Mar 2026 16:43:30 +0000 Subject: [PATCH 4/4] Added support for OCI image indexes and manifest Added test for dockerv2 kiwix-serve (3.8.1) and ociv1 (3.8.2) Fixes #12 --- CHANGELOG.md | 6 ++++++ src/docker_export/__init__.py | 24 ++++++++++++++++-------- tests/test_digest.py | 10 ++++++++++ 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a7c99c..016fbea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- Support for OCI image indexes and manifest (#12) + ## [1.1.0] - 2024-01-30 ### Added diff --git a/src/docker_export/__init__.py b/src/docker_export/__init__.py index 817418a..ab648ce 100755 --- a/src/docker_export/__init__.py +++ b/src/docker_export/__init__.py @@ -37,6 +37,15 @@ logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") logger = logging.getLogger("docker-export") logging.getLogger("urllib3").setLevel(logging.WARNING) +CT_DOCKER_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json" +CT_DOCKER_MANIFEST = "application/vnd.docker.distribution.manifest.v2+json" +CT_OCI_INDEXES = "application/vnd.oci.image.index.v1+json" +CT_OCI_MANIFEST = "application/vnd.oci.image.manifest.v1+json" +CT_OCI_IMAGE_CONFIG = "application/vnd.oci.image.config.v1+json" +CT_OCI_EMPTY = "application/vnd.oci.image.config.v1+json" +CT_OCI_LAYERS = ( + "application/vnd.oci.image.layer.v1.tar,application/vnd.oci.image.layer.v1.tar+gzip" +) class ImageNotFoundError(Exception): ... @@ -384,7 +393,7 @@ def get_manifests(image: Image, auth: RegistryAuth): f"/manifests/{image.reference}", headers=dict( **auth.headers, - **{"Accept": "application/vnd.docker.distribution.manifest.list.v2+json"}, + **{"Accept": ", ".join([CT_DOCKER_MANIFEST_LIST, CT_OCI_INDEXES])}, ), timeout=REQUEST_TIMEOUT, ) @@ -415,13 +424,12 @@ def get_layers_manifest_for( f"/manifests/{reference}", headers=dict( **auth.headers, - **{"Accept": "application/vnd.docker.distribution.manifest.v2+json"}, + **{"Accept": ", ".join([CT_DOCKER_MANIFEST, CT_OCI_MANIFEST])}, ), timeout=REQUEST_TIMEOUT, ) if resp.status_code != http.HTTPStatus.OK: - raise OSError("HTTP {resp.status_code}: {resp.reason} -- {resp.text}") - + raise OSError(f"HTTP {resp.status_code}: {resp.reason} -- {resp.text}") return resp.json() @@ -438,7 +446,7 @@ def get_layers_from_v1_manifest( ) return { - "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "mediaType": CT_DOCKER_MANIFEST, "schemaVersion": 2, "config": { "mediaType": "application/vnd.docker.container.image.v1+json", @@ -447,7 +455,7 @@ def get_layers_from_v1_manifest( }, "layers": [ { - "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "mediaType": CT_DOCKER_MANIFEST, "digest": layer["blobSum"], # "size": None, "platform": {"architecture": architecture, "os": os}, @@ -524,7 +532,7 @@ def download_layer_blob( f"https://{image.registry}/v2/{image.fullname}/blobs/{layer_digest}", headers=dict( **auth.headers, - **{"Accept": "application/vnd.docker.distribution.manifest.v2+json"}, + **{"Accept": CT_DOCKER_MANIFEST}, ), stream=True, timeout=REQUEST_TIMEOUT, @@ -536,7 +544,7 @@ def download_layer_blob( layer["urls"][0], headers=dict( **auth.headers, - **{"Accept": "application/vnd.docker.distribution.manifest.v2+json"}, + **{"Accept": CT_DOCKER_MANIFEST}, ), stream=True, timeout=REQUEST_TIMEOUT, diff --git a/tests/test_digest.py b/tests/test_digest.py index a36f3b3..dc93aa8 100644 --- a/tests/test_digest.py +++ b/tests/test_digest.py @@ -24,6 +24,16 @@ # which is the Docker-Content-Digest "sha256:148e55d6f2c3fa74bdba8f2b6870677cfcb268e5a9bebc9e5135a026f502f447", ), + ( + "ghcr.io/offspot/kiwix-serve:3.8.2", + "linux/arm64", + "sha256:eb186010ca6318da285db02383d2bb4aef45034faead4eb8a78fcde758d919c3", + ), + ( + "ghcr.io/offspot/kiwix-serve:3.8.1", + "linux/amd64", + "sha256:c7e75fd985a93aedcc4582751f23d64bcf4274294fff63c3b7a0ba32bc3d103f", + ), ( f"{kiwix_tools}:3.5.0-2", "linux/amd64",