diff --git a/exasol/toolbox/tools/template.py b/exasol/toolbox/tools/template.py index 14829ea1f..1fcc28568 100644 --- a/exasol/toolbox/tools/template.py +++ b/exasol/toolbox/tools/template.py @@ -2,7 +2,6 @@ import io from collections.abc import Mapping from contextlib import ExitStack -from inspect import cleandoc from pathlib import Path from typing import ( Any, @@ -10,12 +9,11 @@ import importlib_resources as resources import typer -import yaml -from jinja2 import Environment from rich.columns import Columns from rich.console import Console from rich.syntax import Syntax +from exasol.toolbox.util.workflows.workflow import Workflow from noxconfig import PROJECT_CONFIG stdout = Console() @@ -23,10 +21,6 @@ CLI = typer.Typer() -jinja_env = Environment( - variable_start_string="((", variable_end_string="))", autoescape=True -) - def _templates(pkg: str) -> Mapping[str, Any]: def _normalize(name: str) -> str: @@ -72,18 +66,13 @@ def show_templates( def _render_template( src: str | Path, - stack: ExitStack, ) -> str: - input_file = stack.enter_context(open(src, encoding="utf-8")) - - # dynamically render the template with Jinja2 - template = jinja_env.from_string(input_file.read()) - rendered_string = template.render(PROJECT_CONFIG.github_template_dict) - - # validate that the rendered content is a valid YAML. This is not - # written out as by default it does not give GitHub-safe output. - yaml.safe_load(rendered_string) - return cleandoc(rendered_string) + "\n" + src_path = Path(src) + github_template_dict = PROJECT_CONFIG.github_template_dict + workflow = Workflow.load_from_template( + file_path=src_path, github_template_dict=github_template_dict + ) + return workflow.content + "\n" def diff_template(template: str, dest: Path, pkg: str, template_type: str) -> None: @@ -107,7 +96,7 @@ def diff_template(template: str, dest: Path, pkg: str, template_type: str) -> No old = old.read().split("\n") new = new.read().split("\n") elif template_type == "workflow": - new = _render_template(src=new, stack=stack) + new = _render_template(src=new) old = old.read().split("\n") new = new.split("\n") @@ -134,7 +123,7 @@ def _install_template( return output_file = stack.enter_context(open(dest, "wb")) - rendered_string = _render_template(src=src, stack=stack) + rendered_string = _render_template(src=src) output_file.write(rendered_string.encode("utf-8")) diff --git a/exasol/toolbox/util/dependencies/poetry_dependencies.py b/exasol/toolbox/util/dependencies/poetry_dependencies.py index e56ecefe0..f13387091 100644 --- a/exasol/toolbox/util/dependencies/poetry_dependencies.py +++ b/exasol/toolbox/util/dependencies/poetry_dependencies.py @@ -38,7 +38,7 @@ class PoetryToml(BaseModel): def load_from_toml(cls, working_directory: Path) -> PoetryToml: file_path = working_directory / PoetryFiles.pyproject_toml if not file_path.exists(): - raise ValueError(f"File not found: {file_path}") + raise FileExistsError(f"File not found: {file_path}") try: text = file_path.read_text() diff --git a/exasol/toolbox/util/workflows/__init__.py b/exasol/toolbox/util/workflows/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/exasol/toolbox/util/workflows/format_yaml.py b/exasol/toolbox/util/workflows/format_yaml.py new file mode 100644 index 000000000..9f22bec50 --- /dev/null +++ b/exasol/toolbox/util/workflows/format_yaml.py @@ -0,0 +1,60 @@ +import re + +from yaml import SafeDumper +from yaml.resolver import Resolver + +# Regex for common strings in YAML that lose quotes: +# 1. Version numbers (e.g., 2.3.0, 3.10) +# 2. OS/image names (e.g., ubuntu-24.04) +# 3. Numeric strings that look like octals or floats (e.g., 045, 1.2) +QUOTE_REGEX = re.compile(r"^(\d+\.\d+(\.\d+)?|[a-zA-Z]+-\d+\.\d+|0\d+)$") + +# yaml uses a shorthand to identify "on" and "off" tags. +# for GitHub workflows, we do NOT want "on" replaced with "True". +for character in ["O", "o"]: + Resolver.yaml_implicit_resolvers[character] = [ + x + for x in Resolver.yaml_implicit_resolvers[character] + if x[0] != "tag:yaml.org,2002:bool" + ] + + +class GitHubDumper(SafeDumper): + pass + + +def empty_representer(dumper: SafeDumper, data): + """ + Leave empty fields like empty, instead of adding "null" + + Without using `empty_representer` + on: + workflow_call: null + + Using `empty_representer` + on: + workflow_call: + """ + return dumper.represent_scalar("tag:yaml.org,2002:null", "") + + +def str_presenter(dumper: SafeDumper, data): + """ + Present str in a custom format compatible with GitHub + """ + # For line breaks in a multiline step, use pipe "|" instead of quotes "'" + if "\n" in data: + # Ensure it ends with \n so PyYAML doesn't add the '-' strip indicator + if not data.endswith("\n"): + data += "\n" + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + + # For strings with versions, ensure that they are quoted '"' so that they + # are not incorrectly parsed in the workflow, e.g. to an integer instead of a float. + if QUOTE_REGEX.match(data): + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style='"') + return dumper.represent_scalar("tag:yaml.org,2002:str", data) + + +GitHubDumper.add_representer(str, str_presenter) +GitHubDumper.add_representer(type(None), empty_representer) diff --git a/exasol/toolbox/util/workflows/workflow.py b/exasol/toolbox/util/workflows/workflow.py new file mode 100644 index 000000000..fe65cede9 --- /dev/null +++ b/exasol/toolbox/util/workflows/workflow.py @@ -0,0 +1,59 @@ +from inspect import cleandoc +from pathlib import Path +from typing import Any + +from jinja2 import Environment +from pydantic import ( + BaseModel, + ConfigDict, +) +from yaml import ( + dump, + safe_load, +) + +from exasol.toolbox.util.workflows.format_yaml import GitHubDumper + +jinja_env = Environment( + variable_start_string="((", variable_end_string="))", autoescape=True +) + + +def _render_template(template: str, github_template_dict: dict[str, Any]) -> str: + """ + Render the template with Jinja2 & dump as a str + """ + # Dynamically render the template with Jinja2 + jinja_template = jinja_env.from_string(template) + rendered_string = jinja_template.render(github_template_dict) + + # Also checks that the rendered template is a valid YAML. + data = safe_load(rendered_string) + + return cleandoc( + dump( + data, + Dumper=GitHubDumper, + sort_keys=False, # if True, then re-orders the jobs alphabetically + ) + ) + + +class Workflow(BaseModel): + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + content: str + + @classmethod + def load_from_template(cls, file_path: Path, github_template_dict: dict[str, Any]): + if not file_path.exists(): + raise FileNotFoundError(file_path) + + try: + raw_content = file_path.read_text() + rendered_content = _render_template( + template=raw_content, github_template_dict=github_template_dict + ) + return cls(content=rendered_content) + except Exception as e: + raise ValueError(f"Error rendering file: {str(e)}") diff --git a/test/unit/util/workflows/format_yaml_test.py b/test/unit/util/workflows/format_yaml_test.py new file mode 100644 index 000000000..d4f53c8b5 --- /dev/null +++ b/test/unit/util/workflows/format_yaml_test.py @@ -0,0 +1,100 @@ +from inspect import cleandoc + +from yaml import ( + dump, + safe_load, +) + +from exasol.toolbox.util.workflows.format_yaml import GitHubDumper + + +class TestEmptyRepresenter: + documentation = """ + name: Merge-Gate + on: + workflow_call: + """ + + def test_works_as_expected(self): + data = safe_load(cleandoc(self.documentation)) + output = dump( + data, + Dumper=GitHubDumper, + ) + assert output == cleandoc(self.documentation) + "\n" + + def test_default_behavior_differs(self): + expected = cleandoc( + """ + name: Merge-Gate + on: + workflow_call: null + """ + ) + + data = safe_load(cleandoc(self.documentation)) + + output = dump(data) + assert output == expected + "\n" + + +class TestStrPresenter: + doc_with_line_break = """ + steps: + - name: Generate GitHub Summary + run: | + echo -e "# Summary" >> $GITHUB_STEP_SUMMARY + poetry run -- nox -s project:report -- --format markdown >> $GITHUB_STEP_SUMMARY + """ + doc_with_version = """ + steps: + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v5 + with: + python-version: "3.10" + poetry-version: "2.3.0" + """ + + def test_line_break_works_as_expected(self): + data = safe_load(cleandoc(self.doc_with_line_break)) + output = dump( + data, + Dumper=GitHubDumper, + ) + assert output == cleandoc(self.doc_with_line_break) + "\n" + + def test_line_break_with_default_differs(self): + data = safe_load(cleandoc(self.doc_with_line_break)) + output = dump(data) + assert output == ( + "steps:\n" + "- name: Generate GitHub Summary\n" + ' run: \'echo -e "# Summary" >> $GITHUB_STEP_SUMMARY\n' + "\n" + " poetry run -- nox -s project:report -- --format markdown >> " + "$GITHUB_STEP_SUMMARY'\n" + ) + + def test_quote_regex_works_as_expected(self): + data = safe_load(cleandoc(self.doc_with_version)) + output = dump( + data, + Dumper=GitHubDumper, + sort_keys=False, # if True, then re-orders the jobs alphabetically + ) + assert output == cleandoc(self.doc_with_version) + "\n" + + def test_quote_regex_with_default_differs(self): + data = safe_load(cleandoc(self.doc_with_version)) + output = dump( + data, + sort_keys=False, # if True, then re-orders the jobs alphabetically + ) + assert output == ( + "steps:\n" + "- name: Setup Python & Poetry Environment\n" + " uses: exasol/python-toolbox/.github/actions/python-environment@v5\n" + " with:\n" + " python-version: '3.10'\n" + " poetry-version: 2.3.0\n" + ) diff --git a/test/unit/tools/test_template.py b/test/unit/util/workflows/workflow_test.py similarity index 63% rename from test/unit/tools/test_template.py rename to test/unit/util/workflows/workflow_test.py index 354695805..41ac36b36 100644 --- a/test/unit/tools/test_template.py +++ b/test/unit/util/workflows/workflow_test.py @@ -1,12 +1,9 @@ -from contextlib import ExitStack from inspect import cleandoc import pytest -from yaml.parser import ParserError -from exasol.toolbox.tools.template import ( - _render_template, -) +from exasol.toolbox.util.workflows.workflow import Workflow +from noxconfig import PROJECT_CONFIG TEMPLATE = """ name: Publish Documentation @@ -60,43 +57,37 @@ """ -RENDERED_TEMPLATE = """ +WORKFLOW = """ name: Publish Documentation - on: workflow_call: workflow_dispatch: - jobs: - build-documentation: runs-on: "ubuntu-24.04" permissions: contents: read steps: - - name: SCM Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Setup Python & Poetry Environment - uses: exasol/python-toolbox/.github/actions/python-environment@v5 - with: - python-version: "3.10" - poetry-version: "2.3.0" - - - name: Build Documentation - run: | - poetry run -- nox -s docs:multiversion - mv .html-documentation html-documentation - - - name: Upload artifact - uses: actions/upload-pages-artifact@v4 - with: - path: html-documentation - + - name: SCM Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v5 + with: + python-version: "3.10" + poetry-version: "2.3.0" + - name: Build Documentation + run: | + poetry run -- nox -s docs:multiversion + mv .html-documentation html-documentation + - name: Upload artifact + uses: actions/upload-pages-artifact@v4 + with: + path: html-documentation deploy-documentation: - needs: [ build-documentation ] + needs: + - build-documentation permissions: contents: read pages: write @@ -106,9 +97,9 @@ url: ${{ steps.deployment.outputs.page_url }} runs-on: "ubuntu-24.04" steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 """ @@ -131,19 +122,23 @@ """ -class TestRenderTemplate: +class TestWorkflow: @staticmethod def test_works_as_expected(tmp_path): file_path = tmp_path / "test.yml" file_path.write_text(TEMPLATE) - with ExitStack() as stack: - rendered_str = _render_template(src=file_path, stack=stack) - assert rendered_str == cleandoc(RENDERED_TEMPLATE) + "\n" + workflow = Workflow.load_from_template( + file_path=file_path, + github_template_dict=PROJECT_CONFIG.github_template_dict, + ) + assert workflow.content == cleandoc(WORKFLOW) @staticmethod def test_fails_when_yaml_malformed(tmp_path): file_path = tmp_path / "test.yaml" file_path.write_text(BAD_TEMPLATE) - with pytest.raises(ParserError, match="while parsing a block collection"): - with ExitStack() as stack: - _render_template(src=file_path, stack=stack) + with pytest.raises(ValueError, match="while parsing a block collection"): + Workflow.load_from_template( + file_path=file_path, + github_template_dict=PROJECT_CONFIG.github_template_dict, + )