Refactoring/686 output yaml values

.github/workflows/build-and-publish.yml

@@ -4,7 +4,7 @@ on:
   workflow_call:
     secrets:
       PYPI_TOKEN:
-          required: true
+        required: true
 
 jobs:
 
@@ -35,7 +35,7 @@ jobs:
       - name: GitHub Release
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: >
+        run: >-
           gh release create ${GITHUB_REF_NAME}
           --title ${GITHUB_REF_NAME}
           --notes-file ./doc/changes/changes_${GITHUB_REF_NAME}.md

.github/workflows/cd.yml

@@ -15,7 +15,8 @@ jobs:
       contents: read
 
   cd-job:
-    needs: [ check-tag-version-job ]
+    needs:
+      - check-tag-version-job
     name: Continuous Delivery
     uses: ./.github/workflows/build-and-publish.yml
     permissions:
@@ -24,7 +25,8 @@ jobs:
       PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
 
   publish-docs:
-    needs: [ cd-job ]
+    needs:
+      - cd-job
     name: Publish Documentation
     uses: ./.github/workflows/gh-pages.yml
     permissions:

.github/workflows/ci.yml

@@ -2,7 +2,7 @@ name: CI
 
 on:
   pull_request:
-      types: [opened, synchronize, reopened]
+    types: [opened, synchronize, reopened]
   schedule:
     # At 00:00 on every 7th day-of-month from 1 through 31. (https://crontab.guru)
     - cron: "0 0 1/7 * *"
@@ -16,7 +16,8 @@ jobs:
       contents: read
 
   Metrics:
-    needs: [ CI ]
+    needs:
+      - CI
     uses: ./.github/workflows/report.yml
     secrets: inherit
     permissions:

.github/workflows/gh-pages.yml

@@ -33,7 +33,8 @@ jobs:
           path: html-documentation
 
   deploy-documentation:
-    needs: [ build-documentation ]
+    needs:
+      - build-documentation
     permissions:
       contents: read
       pages: write

.github/workflows/merge-gate.yml

@@ -27,7 +27,8 @@ jobs:
 
   slow-checks:
     name: Slow
-    needs: [ run-slow-tests ]
+    needs:
+      - run-slow-tests
     uses: ./.github/workflows/slow-checks.yml
     secrets: inherit
     permissions:
@@ -40,10 +41,11 @@ jobs:
     permissions:
       contents: read
     # If you need additional jobs to be part of the merge gate, add them below
-    needs: [ fast-checks, slow-checks ]
+    needs:
+      - fast-checks
+      - slow-checks
 
     # Each job requires a step, so we added this dummy step.
     steps:
       - name: Approve
-        run: |
-          echo "Merge Approved"
+        run: echo "Merge Approved"

.github/workflows/pr-merge.yml

@@ -26,7 +26,8 @@ jobs:
       id-token: write
 
   metrics:
-    needs: [ ci-job ]
+    needs:
+      - ci-job
     uses: ./.github/workflows/report.yml
     secrets: inherit
     permissions:

.github/workflows/report.yml

@@ -50,7 +50,7 @@ jobs:
           path: metrics.json
 
       - name: Generate GitHub Summary
-        run: |
+        run: |-
           echo -e "# Summary\n" >> $GITHUB_STEP_SUMMARY
           poetry run -- nox -s project:report -- --format markdown >> $GITHUB_STEP_SUMMARY
           poetry run -- nox -s dependency:licenses >> $GITHUB_STEP_SUMMARY

.github/workflows/slow-checks.yml

@@ -12,7 +12,8 @@ jobs:
 
   tests:
     name: Integration-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}})
-    needs: [ build-matrix ]
+    needs:
+      - build-matrix
     runs-on: "ubuntu-24.04"
     permissions:
       contents: read
@@ -44,7 +45,8 @@ jobs:
 
 
   verify-poetry-installation:
-    needs: [ build-matrix ]
+    needs:
+     - build-matrix
     # This Job verifies if pipx installation is successful on each of the
     # selected GitHub Runners.
     strategy:

doc/changes/unreleased.md

@@ -5,3 +5,7 @@
 ## Documentation
 
 * #585: Added instructions how to ignore sonar issues to the User Guide
+
+## Refactoring
+
+* #686: Switched GitHub templates to be fully parsed by ruamel-yaml

exasol/toolbox/templates/github/workflows/build-and-publish.yml

@@ -4,7 +4,7 @@ on:
   workflow_call:
     secrets:
       PYPI_TOKEN:
-          required: true
+        required: true
 
 jobs:
 

exasol/toolbox/templates/github/workflows/cd.yml

@@ -15,7 +15,8 @@ jobs:
       contents: read
 
   cd-job:
-    needs: [ check-tag-version-job ]
+    needs:
+     - check-tag-version-job
     name: Continuous Delivery
     uses: ./.github/workflows/build-and-publish.yml
     permissions:
@@ -24,7 +25,8 @@ jobs:
       PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
 
   publish-docs:
-    needs: [ cd-job ]
+    needs:
+     - cd-job
     name: Publish Documentation
     uses: ./.github/workflows/gh-pages.yml
     permissions:

exasol/toolbox/templates/github/workflows/ci.yml

@@ -2,7 +2,7 @@ name: CI
 
 on:
   pull_request:
-      types: [opened, synchronize, reopened]
+    types: [opened, synchronize, reopened]
   schedule:
     # At 00:00 on every 7th day-of-month from 1 through 31. (https://crontab.guru)
     - cron: "0 0 1/7 * *"
@@ -16,7 +16,8 @@ jobs:
       contents: read
 
   Metrics:
-    needs: [ CI ]
+    needs:
+     - CI
     uses: ./.github/workflows/report.yml
     secrets: inherit
     permissions:

exasol/toolbox/templates/github/workflows/gh-pages.yml

@@ -33,7 +33,8 @@ jobs:
           path: html-documentation
 
   deploy-documentation:
-    needs: [ build-documentation ]
+    needs:
+     - build-documentation
     permissions:
       contents: read
       pages: write

exasol/toolbox/templates/github/workflows/merge-gate.yml

@@ -27,7 +27,8 @@ jobs:
 
   slow-checks:
     name: Slow
-    needs: [ run-slow-tests ]
+    needs:
+     - run-slow-tests
     uses: ./.github/workflows/slow-checks.yml
     secrets: inherit
     permissions:
@@ -40,10 +41,11 @@ jobs:
     permissions:
       contents: read
     # If you need additional jobs to be part of the merge gate, add them below
-    needs: [ fast-checks, slow-checks ]
+    needs:
+     - fast-checks
+     - slow-checks
 
     # Each job requires a step, so we added this dummy step.
     steps:
       - name: Approve
-        run: |
-          echo "Merge Approved"
+        run: echo "Merge Approved"

exasol/toolbox/templates/github/workflows/pr-merge.yml

@@ -26,7 +26,8 @@ jobs:
       id-token: write
 
   metrics:
-    needs: [ ci-job ]
+    needs:
+     - ci-job
     uses: ./.github/workflows/report.yml
     secrets: inherit
     permissions:

exasol/toolbox/templates/github/workflows/slow-checks.yml

@@ -12,7 +12,8 @@ jobs:
 
   tests:
     name: Integration-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}})
-    needs: [ build-matrix ]
+    needs:
+      - build-matrix
     runs-on: "(( os_version ))"
     permissions:
       contents: read

exasol/toolbox/tools/template.py

@@ -2,31 +2,25 @@
 import io
 from collections.abc import Mapping
 from contextlib import ExitStack
-from inspect import cleandoc
 from pathlib import Path
 from typing import (
     Any,
 )
 
 import importlib_resources as resources
 import typer
-import yaml
-from jinja2 import Environment
 from rich.columns import Columns
 from rich.console import Console
 from rich.syntax import Syntax
 
+from exasol.toolbox.util.workflows.workflow import Workflow
 from noxconfig import PROJECT_CONFIG
 
 stdout = Console()
 stderr = Console(stderr=True)
 
 CLI = typer.Typer()
 
-jinja_env = Environment(
-    variable_start_string="((", variable_end_string="))", autoescape=True
-)
-
 
 def _templates(pkg: str) -> Mapping[str, Any]:
     def _normalize(name: str) -> str:
@@ -72,18 +66,13 @@ def show_templates(
 
 def _render_template(
     src: str | Path,
-    stack: ExitStack,
 ) -> str:
-    input_file = stack.enter_context(open(src, encoding="utf-8"))
-
-    # dynamically render the template with Jinja2
-    template = jinja_env.from_string(input_file.read())
-    rendered_string = template.render(PROJECT_CONFIG.github_template_dict)
-
-    # validate that the rendered content is a valid YAML. This is not
-    # written out as by default it does not give GitHub-safe output.
-    yaml.safe_load(rendered_string)
-    return cleandoc(rendered_string) + "\n"
+    src_path = Path(src)
+    github_template_dict = PROJECT_CONFIG.github_template_dict
+    workflow = Workflow.load_from_template(
+        file_path=src_path, github_template_dict=github_template_dict
+    )
+    return workflow.content + "\n"
 
 
 def diff_template(template: str, dest: Path, pkg: str, template_type: str) -> None:
@@ -107,7 +96,7 @@ def diff_template(template: str, dest: Path, pkg: str, template_type: str) -> No
                     old = old.read().split("\n")
                     new = new.read().split("\n")
                 elif template_type == "workflow":
-                    new = _render_template(src=new, stack=stack)
+                    new = _render_template(src=new)
                     old = old.read().split("\n")
                     new = new.split("\n")
 
@@ -134,7 +123,7 @@ def _install_template(
             return
 
         output_file = stack.enter_context(open(dest, "wb"))
-        rendered_string = _render_template(src=src, stack=stack)
+        rendered_string = _render_template(src=src)
         output_file.write(rendered_string.encode("utf-8"))
 
 

exasol/toolbox/util/workflows/template.py

@@ -0,0 +1,44 @@
+from dataclasses import dataclass
+from typing import Any
+
+from jinja2 import Environment
+
+jinja_env = Environment(
+    variable_start_string="((", variable_end_string="))", autoescape=True
+)
+
+import io
+from inspect import cleandoc
+
+from ruamel.yaml import YAML
+
+
+@dataclass(frozen=True)
+class TemplateToWorkflow:
+    template_str: str
+    github_template_dict: dict[str, Any]
+
+    def _render_with_jinja(self, input_str: str) -> str:
+        """
+        Render the template with Jinja.
+        """
+        jinja_template = jinja_env.from_string(input_str)
+        return jinja_template.render(self.github_template_dict)
+
+    def convert(self) -> str:
+        """
+        Convert a workflow template to a rendered workflow that works for GitHub.
+        """
+        yaml = YAML()
+        yaml.width = 200
+        yaml.preserve_quotes = True
+        yaml.sort_base_mapping_type_on_output = False  # type: ignore
+        yaml.indent(mapping=2, sequence=4, offset=2)
+
+        workflow_string = self._render_with_jinja(self.template_str)
+        workflow_dict = yaml.load(workflow_string)
+
+        stream = io.StringIO()
+        yaml.dump(workflow_dict, stream)
+        workflow_string = stream.getvalue()
+        return cleandoc(workflow_string)

exasol/toolbox/util/workflows/workflow.py

@@ -0,0 +1,30 @@
+from pathlib import Path
+from typing import Any
+
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+)
+
+from exasol.toolbox.util.workflows.template import TemplateToWorkflow
+
+
+class Workflow(BaseModel):
+    model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True)
+
+    content: str
+
+    @classmethod
+    def load_from_template(cls, file_path: Path, github_template_dict: dict[str, Any]):
+        if not file_path.exists():
+            raise FileNotFoundError(file_path)
+
+        try:
+            raw_content = file_path.read_text()
+            template_to_workflow = TemplateToWorkflow(
+                template_str=raw_content, github_template_dict=github_template_dict
+            )
+            workflow = template_to_workflow.convert()
+            return cls(content=workflow)
+        except Exception as e:
+            raise ValueError(f"Error rendering file: {str(e)}")