diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 61f5764f..2e16b371 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -4,7 +4,7 @@ description: Installs the given GardenLinux Python library inputs: version: description: GardenLinux Python library version - default: "0.10.16" + default: "0.10.19" python_version: description: Python version to setup default: "3.13" diff --git a/poetry.lock b/poetry.lock index 676e81ee..4346e6ce 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand. [[package]] name = "alabaster" diff --git a/pyproject.toml b/pyproject.toml index efa07c38..dfaf6bde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gardenlinux" -version = "0.10.16" +version = "0.10.19" description = "Contains tools to work with the features directory of gardenlinux, for example deducting dependencies from feature sets or validating cnames" authors = ["Garden Linux Maintainers "] license = "Apache-2.0" diff --git a/src/gardenlinux/constants.py b/src/gardenlinux/constants.py index fb242093..970b6dbc 100644 --- a/src/gardenlinux/constants.py +++ b/src/gardenlinux/constants.py @@ -159,7 +159,7 @@ RELEASE_ID_FILE = ".github_release_id" -REQUESTS_TIMEOUTS = (5, 30) # connect, read +REQUESTS_TIMEOUTS = (5, 60) # connect, read S3_DOWNLOADS_DIR = Path(os.path.dirname(__file__)) / ".." / "s3_downloads" @@ -167,6 +167,10 @@ GLVD_BASE_URL = "https://security.gardenlinux.org/v1" PODMAN_CONNECTION_MAX_IDLE_SECONDS = 3 +PODMAN_FS_CHANGE_ADDED = "added" +PODMAN_FS_CHANGE_DELETED = "deleted" +PODMAN_FS_CHANGE_MODIFIED = "modified" +PODMAN_FS_CHANGE_UNSUPPORTED = "unsupported" # https://github.com/gardenlinux/gardenlinux/issues/3044 # Empty string is the 'legacy' variant with traditional root fs and still needed/supported diff --git a/src/gardenlinux/features/reproducibility/comparator.py b/src/gardenlinux/features/reproducibility/comparator.py index f2b03976..0ff861e7 100644 --- a/src/gardenlinux/features/reproducibility/comparator.py +++ b/src/gardenlinux/features/reproducibility/comparator.py @@ -9,11 +9,15 @@ import importlib.resources import json import re -import tarfile -import tempfile from os import PathLike from pathlib import Path -from typing import Optional + +from ...constants import ( + PODMAN_FS_CHANGE_ADDED, + PODMAN_FS_CHANGE_DELETED, + PODMAN_FS_CHANGE_MODIFIED, +) +from ...oci import Image, Podman, PodmanContext class Comparator(object): @@ -29,120 +33,27 @@ class Comparator(object): Apache License, Version 2.0 """ - _default_whitelist: list[str] = [] - - _nightly_whitelist = json.loads( - importlib.resources.read_text(__name__, "nightly_whitelist.json") - ) - - def __init__( - self, nightly: bool = False, whitelist: list[str] = _default_whitelist - ): + def __init__(self, nightly: bool = False, whitelist: list[str] = []): """ Constructor __init__(Comparator) :param nightly: Flag indicating if the nightlywhitelist should be used :param whitelst: Additional whitelist - :since: 1.0.0 - """ - self.whitelist = whitelist - if nightly: - self.whitelist += self._nightly_whitelist - - @staticmethod - def _unpack(file: PathLike[str]) -> tempfile.TemporaryDirectory[str]: - """ - Unpack a .tar archive or .oci image into a temporary dictionary - - :param file: .tar or .oci file - - :return: TemporaryDirectory Temporary directory containing the unpacked file :since: 1.0.0 """ - output_dir = tempfile.TemporaryDirectory() - file = Path(file).resolve() - if file.name.endswith(".oci"): - with tempfile.TemporaryDirectory() as extracted: - # Extract .oci file - with tarfile.open(file, "r") as tar: - tar.extractall( - path=extracted, filter="fully_trusted", members=tar.getmembers() - ) - - layers_dir = Path(extracted).joinpath("blobs/sha256") - assert layers_dir.is_dir() - - with open(Path(extracted).joinpath("index.json"), "r") as f: - index = json.load(f) - - # Only support first manifest - manifest = index["manifests"][0]["digest"].split(":")[1] - - with open(layers_dir.joinpath(manifest), "r") as f: - manifest = json.load(f) - - layers = [layer["digest"].split(":")[1] for layer in manifest["layers"]] - - # Extract layers in order - for layer in layers: - layer_path = layers_dir.joinpath(layer) - if tarfile.is_tarfile(layer_path): - with tarfile.open(layer_path, "r") as tar: - for member in tar.getmembers(): - try: - tar.extract( - member, - path=output_dir.name, - filter="fully_trusted", - ) - except tarfile.AbsoluteLinkError: - # Convert absolute link to relative link - member.linkpath = ( - "../" * member.path.count("/") - + member.linkpath[1:] - ) - tar.extract( - member, - path=output_dir.name, - filter="fully_trusted", - ) - except tarfile.TarError as e: - print(f"Skipping {member.name} due to error: {e}") - else: - with tarfile.open(file, "r") as tar: - tar.extractall( - path=output_dir.name, - filter="fully_trusted", - members=tar.getmembers(), - ) - - return output_dir - - def _diff_files( - self, cmp: filecmp.dircmp[str], left_root: Optional[Path] = None - ) -> list[str]: - """ - Recursively compare files - - :param cmp: Dircmp to recursively compare - :param left_root: Left root to obtain the archive relative path - - :return: list[Path] List of paths with different content - :since: 1.0.0 - """ - - result = [] - if not left_root: - left_root = Path(cmp.left) - for name in cmp.diff_files: - result.append(f"/{Path(cmp.left).relative_to(left_root).joinpath(name)}") - for sub_cmp in cmp.subdirs.values(): - result += self._diff_files(sub_cmp, left_root=left_root) - return result + self.whitelist = whitelist - def generate(self, a: PathLike[str], b: PathLike[str]) -> tuple[list[str], bool]: + if nightly: + self.whitelist += json.loads( + importlib.resources.read_text(__name__, "nightly_whitelist.json") + ) + + @PodmanContext.wrap + def generate( + self, a: PathLike[str], b: PathLike[str], podman: PodmanContext + ) -> tuple[list[str], bool]: """ Compare two .tar/.oci images with each other @@ -156,16 +67,51 @@ def generate(self, a: PathLike[str], b: PathLike[str]) -> tuple[list[str], bool] if filecmp.cmp(a, b, shallow=False): return [], False - with self._unpack(a) as unpacked_a, self._unpack(b) as unpacked_b: - cmp = filecmp.dircmp(unpacked_a, unpacked_b, shallow=False) - - diff_files = self._diff_files(cmp) - - filtered = [ - file - for file in diff_files - if not any(re.match(pattern, file) for pattern in self.whitelist) - ] - whitelist = len(diff_files) != len(filtered) - - return filtered, whitelist + a = Path(a) + a_image_id = None + + b = Path(b) + b_image_id = None + + differences = [] + podman_api = Podman() + + try: + if a.suffix == ".oci": + a_image_id = podman_api.load_oci_archive(a, podman=podman) + elif a.suffix == ".tar": + a_image_id = Image.import_plain_tar(a, podman=podman) + else: + raise RuntimeError(f"Unsupported file type for comparison: {a.name}") + + if b.suffix == ".oci": + b_image_id = podman_api.load_oci_archive(b, podman=podman) + elif b.suffix == ".tar": + b_image_id = Image.import_plain_tar(b, podman=podman) + else: + raise RuntimeError(f"Unsupported file type for comparison: {b.name}") + + image = podman_api.get_image(a_image_id, podman=podman) + + result = image.get_filesystem_changes( + parent_layer_image_id=b_image_id, podman=podman + ) + + differences = ( + result[PODMAN_FS_CHANGE_ADDED] + result[PODMAN_FS_CHANGE_DELETED] + ) + + whitelist = False + + for entry in result[PODMAN_FS_CHANGE_MODIFIED]: + if not any(re.match(pattern, entry) for pattern in self.whitelist): + differences.append(entry) + else: + whitelist = True + finally: + if a_image_id is not None: + podman.images.remove(a_image_id) + if b_image_id is not None: + podman.images.remove(b_image_id) + + return differences, whitelist diff --git a/src/gardenlinux/github/release/__init__.py b/src/gardenlinux/github/release/__init__.py index ac7d6fff..2f5bf875 100644 --- a/src/gardenlinux/github/release/__init__.py +++ b/src/gardenlinux/github/release/__init__.py @@ -1,11 +1,7 @@ -import json import logging -import os import sys -import requests - -from ...constants import RELEASE_ID_FILE, REQUESTS_TIMEOUTS +from ...constants import RELEASE_ID_FILE from ...logger import LoggerSetup from .release import Release @@ -63,47 +59,4 @@ def write_to_release_id_file(release_id: str | int) -> None: sys.exit(1) -def upload_to_github_release_page( - github_owner: str, - github_repo: str, - gardenlinux_release_id: str | int, - file_to_upload: str, - dry_run: bool, -) -> None: - if dry_run: - LOGGER.info( - f"Dry run: would upload {file_to_upload} to release {gardenlinux_release_id} in repo {github_owner}/{github_repo}" - ) - return - - token = os.environ.get("GITHUB_TOKEN") - if not token: - raise ValueError("GITHUB_TOKEN environment variable not set") - - headers = { - "Authorization": f"token {token}", - "Content-Type": "application/octet-stream", - } - - upload_url = f"https://uploads.github.com/repos/{github_owner}/{github_repo}/releases/{gardenlinux_release_id}/assets?name={os.path.basename(file_to_upload)}" - - try: - with open(file_to_upload, "rb") as f: - file_contents = f.read() - except IOError as e: - LOGGER.error(f"Error reading file {file_to_upload}: {e}") - return - - response = requests.post( - upload_url, headers=headers, data=file_contents, timeout=REQUESTS_TIMEOUTS - ) - if response.status_code == 201: - LOGGER.info("Upload successful") - else: - LOGGER.error( - f"Upload failed with status code {response.status_code}: {response.text}" - ) - response.raise_for_status() - - -__all__ = ["Release", "write_to_release_id_file", "upload_to_github_release_page"] +__all__ = ["Release", "write_to_release_id_file"] diff --git a/src/gardenlinux/github/release/__main__.py b/src/gardenlinux/github/release/__main__.py index 013108b9..18f7e2d2 100644 --- a/src/gardenlinux/github/release/__main__.py +++ b/src/gardenlinux/github/release/__main__.py @@ -6,7 +6,6 @@ from ..release_notes import create_github_release_notes from . import ( - upload_to_github_release_page, write_to_release_id_file, ) from .release import Release @@ -149,6 +148,10 @@ def get_parser() -> argparse.ArgumentParser: help="Perform a dry run without actually uploading the file.", ) + upload_parser.add_argument( + "--overwrite-same-name", action="store_true", default=False + ) + return parser @@ -169,6 +172,7 @@ def main() -> None: body = create_github_release_notes( args.tag, args.commit, GARDENLINUX_GITHUB_RELEASE_BUCKET_NAME ) + if args.dry_run: print("Dry Run ...") print("This release would be created:") @@ -184,9 +188,16 @@ def main() -> None: write_to_release_id_file(f"{release_id}") LOGGER.info(f"Release created with ID: {release_id}") elif args.command == "upload": - upload_to_github_release_page( - args.owner, args.repo, args.release_id, args.file_path, args.dry_run - ) + release = Release.get(args.release_id, repo=args.repo, owner=args.owner) + + if args.dry_run: + print("Dry Run ...") + + print( + f"The file {args.file_path} would be uploaded for release: {release.name}" + ) + else: + release.upload_asset(args.file_path, args.overwrite_same_name) else: parser.print_help() diff --git a/src/gardenlinux/github/release/release.py b/src/gardenlinux/github/release/release.py index d4c5099b..ff663adf 100644 --- a/src/gardenlinux/github/release/release.py +++ b/src/gardenlinux/github/release/release.py @@ -5,7 +5,13 @@ """ from logging import Logger -from typing import Optional +from os import PathLike +from pathlib import Path +from typing import Optional, Self + +from github import GithubException +from github.GitRelease import GitRelease +from github.GitReleaseAsset import GitReleaseAsset from ...logger import LoggerSetup from ..client import Client @@ -19,7 +25,7 @@ class Release(object): :copyright: Copyright 2024 SAP SE :package: gardenlinux :subpackage: github - :since: 1.0.0 + :since: 0.10.19 :license: https://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 """ @@ -39,11 +45,12 @@ def __init__( :param token: GitHub access token :param logger: Logger instance - :since: 1.0.0 + :since: 0.10.19 """ self._owner = owner self._repo = repo + self._release_id: Optional[int] = None self._name: Optional[str] = None self._tag: Optional[str] = None self._commitish: Optional[str] = None @@ -61,10 +68,10 @@ def __init__( @property def body(self) -> str: """ - Returns the Git release body set. + Returns the GitHub release body set. - :return: (str) Git release body - :since: 1.0.0 + :return: (str) GitHub release body + :since: 0.10.19 """ if self._release_body is None: @@ -75,11 +82,11 @@ def body(self) -> str: @body.setter def body(self, value: str) -> None: """ - Sets the Git release body. + Sets the GitHub release body. - :param value: Git release body + :param value: GitHub release body - :since: 1.0.0 + :since: 0.10.19 """ self._release_body = value @@ -90,7 +97,7 @@ def commitish(self) -> Optional[str]: Returns the Git release related commit hash. :return: (str) Git release commit hash - :since: 1.0.0 + :since: 0.10.19 """ return self._commitish @@ -102,18 +109,32 @@ def commitish(self, value: str) -> None: :param value: Git release commit hash - :since: 1.0.0 + :since: 0.10.19 """ self._commitish = value + @property + def id(self) -> int: + """ + Returns the GitHub release ID set. + + :return: (int) GitHub release ID + :since: 0.10.19 + """ + + if self._release_id is None: + raise ValueError("GitHub release ID not set") + + return self._release_id + @property def is_latest(self) -> bool: """ Returns true if the Git release is marked as "latest". :return: (str) Git release latest status - :since: 1.0.0 + :since: 0.10.19 """ return self._latest @@ -125,7 +146,7 @@ def is_latest(self, value: bool) -> None: :param value: Git release latest status - :since: 1.0.0 + :since: 0.10.19 """ self._latest = bool(value) @@ -136,7 +157,7 @@ def is_pre_release(self) -> bool: Returns true if the Git release is marked as pre-release. :return: (str) Git release pre-release status - :since: 1.0.0 + :since: 0.10.19 """ return self._pre_release @@ -148,7 +169,7 @@ def is_pre_release(self, value: bool) -> None: :param value: Git release pre-release status - :since: 1.0.0 + :since: 0.10.19 """ self._pre_release = bool(value) @@ -159,7 +180,7 @@ def name(self) -> str: Returns the Git release name set. :return: (str) Git release name - :since: 1.0.0 + :since: 0.10.19 """ if self._name is None: @@ -174,7 +195,7 @@ def name(self, value: str) -> None: :param value: Git release name - :since: 1.0.0 + :since: 0.10.19 """ self._name = value @@ -185,7 +206,7 @@ def tag(self) -> str: Returns the Git release tag set. :return: (str) Git release tag - :since: 1.0.0 + :since: 0.10.19 """ if self._tag is None: @@ -200,17 +221,39 @@ def tag(self, value: str) -> None: :param value: Git release tag - :since: 1.0.0 + :since: 0.10.19 """ self._tag = value + def _copy_from_release_object(self, release_object: GitRelease | Self) -> None: + """ + Copy values from an given GitHub release. + + :return: (str) GitHub release ID created + :since: 0.10.19 + """ + + self._name = release_object.name + + if isinstance(release_object, GitRelease): + self._release_id = release_object.id + self._tag = release_object.tag_name + self._commitish = release_object.target_commitish + self._pre_release = release_object.prerelease + self._release_body = release_object.body_text + else: + self._tag = release_object.tag + self._commitish = release_object.commitish + self._pre_release = release_object.is_pre_release + self._release_body = release_object.body + def create(self) -> int: """ Creates an GitHub release. :return: (str) GitHub release ID created - :since: 1.0.0 + :since: 0.10.19 """ kwargs = { @@ -228,4 +271,92 @@ def create(self) -> int: f"{self._owner}/{self._repo}" ).create_git_release(self.tag, **kwargs) - return release.id # type: ignore[no-any-return] + self._release_id = release.id + + return self._release_id + + def get_asset_by_name(self, asset_name: str) -> GitReleaseAsset: + """ + Returns an GitHub release asset by the given name. + + :param asset_name: Asset name + + :return: (object) GitHub release asset + :since: 0.10.19 + """ + + github_release = self._client.get_repo( + f"{self._owner}/{self._repo}" + ).get_release(self.id) + + for asset in github_release.assets: + if asset_name == asset.name: + return asset + + raise RuntimeError(f"No asset found with name: {asset_name}") + + def upload_asset( + self, asset_file_path_name: PathLike[str] | str, overwrite: bool = False + ) -> None: + """ + Uploads an GitHub release asset. + + :param asset_file_path_name: File path and name to be uploaded + + :since: 0.10.19 + """ + + if not isinstance(asset_file_path_name, PathLike): + asset_file_path_name = Path(asset_file_path_name) + + if asset_file_path_name.stat().st_size < 1: # type: ignore[attr-defined] + self._logger.info(f"{asset_file_path_name} is empty and will be ignored") + return + + github_release = self._client.get_repo( + f"{self._owner}/{self._repo}" + ).get_release(self.id) + + asset_file_name = asset_file_path_name.name # type: ignore[attr-defined] + + try: + github_release.upload_asset(str(asset_file_path_name), name=asset_file_name) + except GithubException as exc: + is_asset_upload_retried = False + + if overwrite and exc.status == 422: + asset = self.get_asset_by_name(asset_file_name) + + asset.delete_asset() + self.upload_asset(asset_file_path_name) + + is_asset_upload_retried = True + + if not is_asset_upload_retried: + raise + + self._logger.info(f"Uploaded file '{asset_file_name}'") + + @staticmethod + def get( + release_id: int, + repo: str, + owner: str = "gardenlinux", + token: Optional[str] = None, + logger: Optional[Logger] = None, + ) -> "Release": + """ + Creates an GitHub release. + + :return: (str) GitHub release ID created + :since: 0.10.19 + """ + + github_release = ( + Client(token, logger).get_repo(f"{owner}/{repo}").get_release(release_id) + ) + + release = Release(repo, owner, token, logger) + release._copy_from_release_object(github_release) + + return release diff --git a/src/gardenlinux/oci/__init__.py b/src/gardenlinux/oci/__init__.py index d152bb37..35ab1a4e 100644 --- a/src/gardenlinux/oci/__init__.py +++ b/src/gardenlinux/oci/__init__.py @@ -5,6 +5,7 @@ """ from .container import Container +from .image import Image from .image_manifest import ImageManifest from .index import Index from .layer import Layer @@ -15,6 +16,7 @@ __all__ = [ "Container", "ImageManifest", + "Image", "Index", "Layer", "Manifest", diff --git a/src/gardenlinux/oci/image.py b/src/gardenlinux/oci/image.py new file mode 100644 index 00000000..b42e906e --- /dev/null +++ b/src/gardenlinux/oci/image.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- + +""" +OCI podman +""" + +import logging +from os import PathLike +from pathlib import Path +from tarfile import open as tarfile_open +from tempfile import TemporaryDirectory +from typing import Any, Dict, List, Optional +from urllib.parse import urlencode + +from podman.domain.images import Image as _Image + +from ..constants import ( + PODMAN_FS_CHANGE_ADDED, + PODMAN_FS_CHANGE_DELETED, + PODMAN_FS_CHANGE_MODIFIED, + PODMAN_FS_CHANGE_UNSUPPORTED, +) +from .podman_context import PodmanContext +from .podman_object_context import PodmanObjectContext + +PODMAN_CHANGES_KINDS = { + 0: PODMAN_FS_CHANGE_MODIFIED, + 1: PODMAN_FS_CHANGE_ADDED, + 2: PODMAN_FS_CHANGE_DELETED, +} + + +class Image(PodmanObjectContext): + """ + Podman image class with extended API features support. + + :author: Garden Linux Maintainers + :copyright: Copyright 2024 SAP SE + :package: gardenlinux + :subpackage: oci + :since: 1.0.0 + :license: https://www.apache.org/licenses/LICENSE-2.0 + Apache License, Version 2.0 + """ + + def __init__(self, image: _Image, logger: Optional[logging.Logger] = None): + """ + Constructor __init__(Image) + + :since: 1.0.0 + """ + + PodmanObjectContext.__init__(self, logger) + self._image_id = image.id + + @property + def id(self) -> str: + """ + podman-py.readthedocs.io: Returns the identifier for the object. + + :return: (str) Identifier for the object + :since: 1.0.0 + """ + + return self._image_id # type: ignore[no-any-return] + + @property + @PodmanContext.wrap + def labels(self, podman: PodmanContext) -> Dict[str, str]: + """ + podman-py.readthedocs.io: Returns the identifier for the object. + + :return: (str) Identifier for the object + :since: 1.0.0 + """ + + return self._get(podman=podman).labels # type: ignore[no-any-return] + + @property + @PodmanContext.wrap + def layer_image_ids(self, podman: PodmanContext) -> List[str]: + """ + Returns the podman image IDs of all parent layers. + + :param podman: Podman context + + :return: (list) Podman layer image IDs + :since: 1.0.0 + """ + + return [ + image_data["Id"] + for image_data in self.history(podman=podman) + if len(image_data["Id"]) == 64 + ] + + def __getattr__( + self, + name: str, + ) -> Any: + """ + python.org: Called when an attribute lookup has not found the attribute in + the usual places (i.e. it is not an instance attribute nor is it found in the + class tree for self). + + :param name: Attribute name + + :return: (mixed) Attribute + :since: 1.0.0 + """ + + @PodmanObjectContext.wrap + def wrapped_context(podman: PodmanContext, *args: Any, **kwargs: Any) -> Any: + """ + Wrapping function to use the podman context. + """ + + py_attr = getattr(self._get(podman=podman), name) + return py_attr(*args, **kwargs) + + return wrapped_context + + def _get(self, podman: PodmanContext) -> _Image: + """ + Returns the underlying podman image object. + + :param podman: Podman context + + :return: (podman.domains.images.Image) Podman image object + :since: 1.0.0 + """ + + return podman.images.get(self._image_id) + + @PodmanContext.wrap + def get_filesystem_changes( + self, podman: PodmanContext, parent_layer_image_id: Optional[str] = None + ) -> Dict[str, List[str]]: + """ + Returns the underlying podman image object. + + :param podman: Podman context + + :return: (_Image) Podman image object + :since: 1.0.0 + """ + + changes: Dict[str, List[str]] = { + PODMAN_FS_CHANGE_ADDED: [], + PODMAN_FS_CHANGE_DELETED: [], + PODMAN_FS_CHANGE_MODIFIED: [], + PODMAN_FS_CHANGE_UNSUPPORTED: [], + } + + query = "" + + if parent_layer_image_id is not None: + query = urlencode({"parent": parent_layer_image_id}) + + resp = self._raw_request( + "get", f"/images/{self._image_id}/changes?{query}", podman=podman + ) + + resp.raise_for_status() + + for entry in resp.json(): + changes[ + PODMAN_CHANGES_KINDS.get(entry["Kind"], PODMAN_FS_CHANGE_UNSUPPORTED) + ].append(entry["Path"]) + + return changes + + @staticmethod + @PodmanContext.wrap + def import_plain_tar(tar_file_name: PathLike[str], podman: PodmanContext) -> str: + """ + Import a plain filesystem tar archive into an OCI image. + + :param tar_file_name: Plain filesystem tar archive + :param podman: Podman context + + :return: (str) Podman image ID + :since: 1.0.0 + """ + + image_id = None + + with TemporaryDirectory() as tmpdir: + container_file_name = Path(tmpdir, "ContainerFile") + tarfile_open(tar_file_name, dereference=True).extractall( + path=Path(tmpdir, "archive_content"), + filter="fully_trusted", + numeric_owner=True, + ) + + with container_file_name.open("w") as container_file: + container_file.write("FROM scratch\nCOPY archive_content/ /") + + image, _ = podman.images.build(path=tmpdir, dockerfile=container_file_name) + image_id = image.id + + return image_id # type: ignore[no-any-return] diff --git a/src/gardenlinux/oci/podman.py b/src/gardenlinux/oci/podman.py index fc44b77b..f66035f9 100644 --- a/src/gardenlinux/oci/podman.py +++ b/src/gardenlinux/oci/podman.py @@ -12,6 +12,7 @@ from typing import Any, Dict, List, Optional from ..logger import LoggerSetup +from .image import Image from .podman_context import PodmanContext @@ -116,12 +117,12 @@ def build_and_save_oci_archive( return {oci_archive_file_name.name: image_id} @PodmanContext.wrap - def get_image_id( + def get_image( self, container: str, podman: PodmanContext, oci_tag: Optional[str] = None, - ) -> str: + ) -> Image: """ Returns the Podman image ID for a given OCI container tag. @@ -136,7 +137,22 @@ def get_image_id( else: container_tag += f":{oci_tag}" - image = podman.images.get(container_tag) + return Image(podman.images.get(container_tag)) + + @PodmanContext.wrap + def get_image_id( + self, + container: str, + podman: PodmanContext, + oci_tag: Optional[str] = None, + ) -> str: + """ + Returns the Podman image ID for a given OCI container tag. + + :since: 1.0.0 + """ + + image = self.get_image(container, oci_tag=oci_tag, podman=podman) return image.id # type: ignore[no-any-return] @PodmanContext.wrap @@ -183,7 +199,7 @@ def pull( podman: PodmanContext, platform: Optional[str] = None, oci_tag: Optional[str] = None, - ) -> None: + ) -> str: """ Pulls a given OCI container. diff --git a/src/gardenlinux/oci/podman_object_context.py b/src/gardenlinux/oci/podman_object_context.py new file mode 100644 index 00000000..deeaa32c --- /dev/null +++ b/src/gardenlinux/oci/podman_object_context.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- + +""" +OCI podman context +""" + +import logging +from functools import wraps +from typing import Any, Optional + +from requests import Response + +from ..logger import LoggerSetup +from .podman_context import PodmanContext + + +class PodmanObjectContext(object): + """ + Podman object context handles access to the podman context for API calls. + + :author: Garden Linux Maintainers + :copyright: Copyright 2024 SAP SE + :package: gardenlinux + :subpackage: oci + :since: 1.0.0 + :license: https://www.apache.org/licenses/LICENSE-2.0 + Apache License, Version 2.0 + """ + + def __init__(self, logger: Optional[logging.Logger] = None): + """ + Constructor __init__(PodmanObjectContext) + + :since: 1.0.0 + """ + + if logger is None or not logger.hasHandlers(): + logger = LoggerSetup.get_logger("gardenlinux.oci") + + self._logger = logger + + @PodmanContext.wrap + def _raw_request( + self, + method: str, + path_and_parameters: str, + podman: PodmanContext, + **kwargs: Any, + ) -> Response: + """ + Returns the podman API response for the request given. + + :param method: Podman API method + :param path_and_parameters: Podman API path and query parameters + :param podman: Podman context + + :return: (Response) Podman API response + :since: 1.0.0 + """ + + method_callable = getattr(podman.api, method) + return method_callable(path_and_parameters, **kwargs) # type: ignore[no-any-return] + + @staticmethod + def wrap(f: Any) -> Any: + """ + Wraps the given function to provide access to a podman client. + + :since: 1.0.0 + """ + + @wraps(f) + @PodmanContext.wrap + def decorator(*args: Any, **kwargs: Any) -> Any: + """ + Decorator for wrapping a function or method with a call context. + """ + + podman = kwargs.get("podman") + + if podman is None: + raise RuntimeError("Podman context not ready") + + del kwargs["podman"] + + return f(podman=podman, *args, **kwargs) + + return decorator diff --git a/tests/github/conftest.py b/tests/github/conftest.py index 114adcfc..8361738c 100644 --- a/tests/github/conftest.py +++ b/tests/github/conftest.py @@ -29,7 +29,7 @@ def github_token() -> Generator[None, None, None]: @pytest.fixture def artifact_for_upload(downloads_dir: None) -> Generator[Path, None, None]: artifact = S3_DOWNLOADS_DIR / "artifact.log" - artifact.touch() + artifact.write_text("Everything is fine so far") yield artifact artifact.unlink() diff --git a/tests/github/constants.py b/tests/github/constants.py index 91ea3419..a562bf8f 100644 --- a/tests/github/constants.py +++ b/tests/github/constants.py @@ -1,3 +1,28 @@ +from ..constants import TEST_COMMIT, TEST_GARDENLINUX_RELEASE + +RELEASE_JSON = { + "url": "https://api.github.com/repos/gardenlinux/gardenlinux/releases/1", + "html_url": f"https://github.com/gardenlinux/gardenlinux/releases/{TEST_GARDENLINUX_RELEASE}", + "assets_url": "https://api.github.com/repos/gardenlinux/gardenlinux/releases/1/assets", + "upload_url": "https://uploads.github.com/repos/gardenlinux/gardenlinux/releases/1/assets{?name,label}", + "tarball_url": "https://api.github.com/repos/gardenlinux/gardenlinux/tarball/{TEST_GARDENLINUX_RELEASE}", + "zipball_url": "https://api.github.com/repos/gardenlinux/gardenlinux/zipball/{TEST_GARDENLINUX_RELEASE}", + "discussion_url": "https://github.com/gardenlinux/gardenlinux/discussions/1", + "id": 1, + "node_id": "MDc6UmVsZWFzZTE=", + "tag_name": TEST_GARDENLINUX_RELEASE, + "target_commitish": TEST_COMMIT, + "name": TEST_GARDENLINUX_RELEASE, + "body": "Happily copied from REST API endpoints for releases @ github.com", + "draft": False, + "prerelease": False, + "immutable": False, + "created_at": "2013-02-27T19:35:32Z", + "published_at": "2013-02-27T19:35:32Z", + "author": {}, + "assets": [], +} + REPO_JSON = { "id": 1, "node_id": "test", diff --git a/tests/github/test_github_script.py b/tests/github/test_github_script.py index c3ca2ede..9b4a062e 100644 --- a/tests/github/test_github_script.py +++ b/tests/github/test_github_script.py @@ -1,4 +1,5 @@ import sys +from pathlib import Path import pytest import requests_mock @@ -7,7 +8,7 @@ from gardenlinux.constants import GARDENLINUX_GITHUB_RELEASE_BUCKET_NAME from ..constants import TEST_GARDENLINUX_COMMIT, TEST_GARDENLINUX_RELEASE -from .constants import REPO_JSON +from .constants import RELEASE_JSON, REPO_JSON def test_script_parse_args_wrong_command( @@ -49,23 +50,6 @@ def test_script_parse_args_create_command_required_args( ) -def test_script_parse_args_upload_command_required_args( - monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] -) -> None: - monkeypatch.setattr( - sys, "argv", ["gh", "upload", "--owner", "gardenlinux", "--repo", "gardenlinux"] - ) - - with pytest.raises(SystemExit): - gh.main() - captured = capfd.readouterr() - - assert ( - "the following arguments are required: --release_id, --file_path" - in captured.err - ), "Expected help message on missing arguments for 'upload' command" - - def test_script_create_dry_run( monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] ) -> None: @@ -146,32 +130,201 @@ def test_script_create( ), "Expected a release creation confirmation log entry" -def test_script_upload_dry_run( +def test_script_upload_needs_github_token( + monkeypatch: pytest.MonkeyPatch, artifact_for_upload: Path +) -> None: + with pytest.raises(ValueError) as exn: + monkeypatch.setattr( + sys, + "argv", + [ + "gh", + "upload", + "--owner", + "gardenlinux", + "--repo", + "gardenlinux", + "--release_id", + TEST_GARDENLINUX_RELEASE, + "--file_path", + str(artifact_for_upload), + "--dry-run", + ], + ) + + gh.main() + + assert str(exn.value) == "GITHUB_TOKEN environment variable not set", ( + "Expected an exception to be raised on missing GITHUB_TOKEN environment variable" + ) + + +def test_script_parse_args_upload_command_required_args( monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] ) -> None: monkeypatch.setattr( - sys, - "argv", - [ - "gh", - "upload", - "--owner", - "gardenlinux", - "--repo", - "gardenlinux", - "--release_id", - TEST_GARDENLINUX_RELEASE, - "--file_path", - "foo", - "--dry-run", - ], - ) - monkeypatch.setattr( - "gardenlinux.github.release.__main__.upload_to_github_release_page", - lambda a1, a2, a3, a4, dry_run: print(f"dry-run: {dry_run}"), + sys, "argv", ["gh", "upload", "--owner", "gardenlinux", "--repo", "gardenlinux"] ) - gh.main() + with pytest.raises(SystemExit): + gh.main() captured = capfd.readouterr() - assert captured.out == "dry-run: True\n" + assert ( + "the following arguments are required: --release_id, --file_path" + in captured.err + ), "Expected help message on missing arguments for 'upload' command" + + +def test_script_upload_dry_run( + monkeypatch: pytest.MonkeyPatch, + capfd: pytest.CaptureFixture[str], + github_token: str, + artifact_for_upload: Path, +) -> None: + with requests_mock.Mocker() as m: + m.get( + "//api.github.com:443/repos/gardenlinux/gardenlinux", + json=REPO_JSON, + status_code=200, + ) + + m.get( + f"//api.github.com:443/repos/gardenlinux/gardenlinux/releases/tags/{TEST_GARDENLINUX_RELEASE}", + json=RELEASE_JSON, + status_code=200, + ) + + monkeypatch.setattr( + sys, + "argv", + [ + "gh", + "upload", + "--owner", + "gardenlinux", + "--repo", + "gardenlinux", + "--release_id", + TEST_GARDENLINUX_RELEASE, + "--file_path", + str(artifact_for_upload), + "--dry-run", + ], + ) + + gh.main() + + captured = capfd.readouterr() + assert "would be uploaded for release" in captured.out, ( + "Expected a dry‑run log entry" + ) + + +def test_script_upload_inaccessible_file( + monkeypatch: pytest.MonkeyPatch, + capfd: pytest.CaptureFixture[str], + github_token: str, + artifact_for_upload: Path, +) -> None: + artifact_for_upload.chmod(0) + + with requests_mock.Mocker() as m: + m.get( + "//api.github.com:443/repos/gardenlinux/gardenlinux", + json=REPO_JSON, + status_code=200, + ) + + m.get( + "//api.github.com:443/repos/gardenlinux/gardenlinux/releases/1", + json=RELEASE_JSON, + status_code=200, + ) + + m.get( + f"//api.github.com:443/repos/gardenlinux/gardenlinux/releases/tags/{TEST_GARDENLINUX_RELEASE}", + json=RELEASE_JSON, + status_code=200, + ) + + m.post( + f"//uploads.github.com:443/repos/gardenlinux/gardenlinux/releases/{TEST_GARDENLINUX_RELEASE}/assets?name=artifact.log", + text="{}", + status_code=201, + ) + + monkeypatch.setattr( + sys, + "argv", + [ + "gh", + "upload", + "--owner", + "gardenlinux", + "--repo", + "gardenlinux", + "--release_id", + TEST_GARDENLINUX_RELEASE, + "--file_path", + str(artifact_for_upload), + ], + ) + + with pytest.raises(PermissionError): + gh.main() + + +def test_script_upload( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + github_token: str, + artifact_for_upload: Path, +) -> None: + with requests_mock.Mocker() as m: + m.get( + "//api.github.com:443/repos/gardenlinux/gardenlinux", + json=REPO_JSON, + status_code=200, + ) + + m.get( + "//api.github.com:443/repos/gardenlinux/gardenlinux/releases/1", + json=RELEASE_JSON, + status_code=200, + ) + + m.get( + f"//api.github.com:443/repos/gardenlinux/gardenlinux/releases/tags/{TEST_GARDENLINUX_RELEASE}", + json=RELEASE_JSON, + status_code=200, + ) + + m.post( + "//uploads.github.com:443/repos/gardenlinux/gardenlinux/releases/1/assets?label=&name=artifact.log", + json={}, + status_code=201, + ) + + monkeypatch.setattr( + sys, + "argv", + [ + "gh", + "upload", + "--owner", + "gardenlinux", + "--repo", + "gardenlinux", + "--release_id", + TEST_GARDENLINUX_RELEASE, + "--file_path", + str(artifact_for_upload), + ], + ) + + gh.main() + + assert any("Uploaded file" in record.message for record in caplog.records), ( + "Expected a upload file log entry" + ) diff --git a/tests/github/test_upload_to_github_release_page.py b/tests/github/test_upload_to_github_release_page.py deleted file mode 100644 index 11d3b976..00000000 --- a/tests/github/test_upload_to_github_release_page.py +++ /dev/null @@ -1,181 +0,0 @@ -import sys -from pathlib import Path - -import pytest -import requests -import requests_mock - -import gardenlinux.github.release.__main__ as gh -from gardenlinux.github.release import upload_to_github_release_page - -from ..constants import TEST_GARDENLINUX_RELEASE - - -def test_upload_to_github_release_page_dryrun( - caplog: pytest.LogCaptureFixture, artifact_for_upload: Path -) -> None: - with requests_mock.Mocker(): - assert ( - upload_to_github_release_page( # type: ignore[func-returns-value] - "gardenlinux", - "gardenlinux", - TEST_GARDENLINUX_RELEASE, - str(artifact_for_upload), - dry_run=True, - ) - is None - ) - assert any( - "Dry run: would upload" in record.message for record in caplog.records - ), "Expected a dry‑run log entry" - - -def test_upload_to_github_release_page_needs_github_token( - downloads_dir: None, artifact_for_upload: Path -) -> None: - with requests_mock.Mocker(): - with pytest.raises(ValueError) as exn: - upload_to_github_release_page( - "gardenlinux", - "gardenlinux", - TEST_GARDENLINUX_RELEASE, - str(artifact_for_upload), - dry_run=False, - ) - assert str(exn.value) == "GITHUB_TOKEN environment variable not set", ( - "Expected an exception to be raised on missing GITHUB_TOKEN environment variable" - ) - - -def test_upload_to_github_release_page( - downloads_dir: None, - caplog: pytest.LogCaptureFixture, - github_token: None, - artifact_for_upload: Path, -) -> None: - with requests_mock.Mocker(real_http=True) as m: - m.post( - f"https://uploads.github.com/repos/gardenlinux/gardenlinux/releases/{TEST_GARDENLINUX_RELEASE}/assets?name=artifact.log", - text="{}", - status_code=201, - ) - - upload_to_github_release_page( - "gardenlinux", - "gardenlinux", - TEST_GARDENLINUX_RELEASE, - str(artifact_for_upload), - dry_run=False, - ) - assert any( - "Upload successful" in record.message for record in caplog.records - ), "Expected an upload confirmation log entry" - - -def test_upload_to_github_release_page_unreadable_artifact( - downloads_dir: None, - caplog: pytest.LogCaptureFixture, - github_token: None, - artifact_for_upload: Path, -) -> None: - artifact_for_upload.chmod(0) - - upload_to_github_release_page( - "gardenlinux", - "gardenlinux", - TEST_GARDENLINUX_RELEASE, - str(artifact_for_upload), - dry_run=False, - ) - assert any("Error reading file" in record.message for record in caplog.records), ( - "Expected an error message log entry" - ) - - -def test_upload_to_github_release_page_failed( - downloads_dir: None, - caplog: pytest.LogCaptureFixture, - github_token: None, - artifact_for_upload: Path, -) -> None: - with requests_mock.Mocker(real_http=True) as m: - m.post( - f"https://uploads.github.com/repos/gardenlinux/gardenlinux/releases/{TEST_GARDENLINUX_RELEASE}/assets?name=artifact.log", - text="{}", - status_code=503, - ) - - with pytest.raises(requests.exceptions.HTTPError): - upload_to_github_release_page( - "gardenlinux", - "gardenlinux", - TEST_GARDENLINUX_RELEASE, - str(artifact_for_upload), - dry_run=False, - ) - assert any( - "Upload failed with status code 503:" in record.message - for record in caplog.records - ), "Expected an error HTTP status code to be logged" - - -def test_script_parse_args_wrong_command( - monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] -) -> None: - monkeypatch.setattr(sys, "argv", ["gh", "rejoice"]) - - with pytest.raises(SystemExit): - gh.main() - captured = capfd.readouterr() - - assert "argument command: invalid choice: 'rejoice'" in captured.err, ( - "Expected help message printed" - ) - - -def test_script_parse_args_upload_command_required_args( - monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] -) -> None: - monkeypatch.setattr( - sys, "argv", ["gh", "upload", "--owner", "gardenlinux", "--repo", "gardenlinux"] - ) - - with pytest.raises(SystemExit): - gh.main() - captured = capfd.readouterr() - - assert ( - "the following arguments are required: --release_id, --file_path" - in captured.err - ), "Expected help message on missing arguments for 'upload' command" - - -def test_script_upload_dry_run( - monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] -) -> None: - monkeypatch.setattr( - sys, - "argv", - [ - "gh", - "upload", - "--owner", - "gardenlinux", - "--repo", - "gardenlinux", - "--release_id", - TEST_GARDENLINUX_RELEASE, - "--file_path", - "foo", - "--dry-run", - ], - ) - monkeypatch.setattr( - "gardenlinux.github.release.__main__.upload_to_github_release_page", - lambda a1, a2, a3, a4, dry_run: print(f"dry-run: {dry_run}"), - ) - - gh.main() - captured = capfd.readouterr() - - assert captured.out == "dry-run: True\n"