From 2ba062ba94c6b6bb60145679691ad308664265f4 Mon Sep 17 00:00:00 2001 From: ckunki Date: Sun, 5 Apr 2026 14:18:13 +0200 Subject: [PATCH 01/21] #402: Created nox task to detect resolved GitHub security issues --- doc/changes/unreleased.md | 1 + exasol/toolbox/nox/_dependencies.py | 19 +++- exasol/toolbox/util/dependencies/audit.py | 2 +- .../dependencies/track_vulnerabilities.py | 91 ++++++++++++++----- .../track_vulnerabilities_test.py | 71 ++++++++------- 5 files changed, 123 insertions(+), 61 deletions(-) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 1a6832109..920f82f7a 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -15,6 +15,7 @@ To ensure usage of secure packages, it is up to the user to similarly relock the ## Features * #740: Added nox session `release:update` +* #402: Created nox task to detect resolved GitHub security issues ## Security Issues diff --git a/exasol/toolbox/nox/_dependencies.py b/exasol/toolbox/nox/_dependencies.py index dc860875a..ffa0ba902 100644 --- a/exasol/toolbox/nox/_dependencies.py +++ b/exasol/toolbox/nox/_dependencies.py @@ -9,17 +9,21 @@ from exasol.toolbox.util.dependencies.audit import ( PipAuditException, Vulnerabilities, + get_vulnerabilities, + get_vulnerabilities_from_latest_tag, ) from exasol.toolbox.util.dependencies.licenses import ( PackageLicenseReport, get_licenses, ) from exasol.toolbox.util.dependencies.poetry_dependencies import get_dependencies +from exasol.toolbox.util.dependencies.track_vulnerabilities import SecurityAudit +from noxconfig import PROJECT_CONFIG @nox.session(name="dependency:licenses", python=False) def dependency_licenses(session: Session) -> None: - """Return the packages with their licenses""" + """Report licenses for all dependencies.""" dependencies = get_dependencies(working_directory=Path()) licenses = get_licenses() license_markdown = PackageLicenseReport( @@ -30,7 +34,7 @@ def dependency_licenses(session: Session) -> None: @nox.session(name="dependency:audit", python=False) def audit(session: Session) -> None: - """Check for known vulnerabilities""" + """Report known vulnerabilities.""" try: vulnerabilities = Vulnerabilities.load_from_pip_audit(working_directory=Path()) @@ -39,3 +43,14 @@ def audit(session: Session) -> None: security_issue_dict = vulnerabilities.security_issue_dict print(json.dumps(security_issue_dict, indent=2)) + + +@nox.session(name="security:audit", python=False) +def security_audit(session: Session) -> None: + """Report resolved vulnerabilities in dependencies.""" + path = PROJECT_CONFIG.root_path + audit = SecurityAudit( + previous_vulnerabilities=get_vulnerabilities_from_latest_tag(path), + current_vulnerabilities=get_vulnerabilities(path), + ) + print(audit.report_resolved_vulnerabilities()) diff --git a/exasol/toolbox/util/dependencies/audit.py b/exasol/toolbox/util/dependencies/audit.py index 53fb67352..783e21ab4 100644 --- a/exasol/toolbox/util/dependencies/audit.py +++ b/exasol/toolbox/util/dependencies/audit.py @@ -177,7 +177,7 @@ def audit_poetry_files(working_directory: Path) -> str: tmpdir = Path(path) (tmpdir / requirements_txt).write_text(output.stdout) - command = ["pip-audit", "-r", requirements_txt, "-f", "json"] + command = ["pip-audit", "--disable-pip", "-r", requirements_txt, "-f", "json"] output = subprocess.run( command, capture_output=True, diff --git a/exasol/toolbox/util/dependencies/track_vulnerabilities.py b/exasol/toolbox/util/dependencies/track_vulnerabilities.py index d9d663797..1afe5d7e0 100644 --- a/exasol/toolbox/util/dependencies/track_vulnerabilities.py +++ b/exasol/toolbox/util/dependencies/track_vulnerabilities.py @@ -1,3 +1,6 @@ +from inspect import cleandoc +from typing import Dict + from pydantic import ( BaseModel, ConfigDict, @@ -6,37 +9,75 @@ from exasol.toolbox.util.dependencies.audit import Vulnerability -class ResolvedVulnerabilities(BaseModel): - model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) - - previous_vulnerabilities: list[Vulnerability] - current_vulnerabilities: list[Vulnerability] +class VulnerabilityMatcher: + def __init__(self, current_vulnerabilities: list[Vulnerability]): + # Dict of current vulnerabilities: + # * keys: package names + # * values: set of each vulnerability's references + self._references = { + v.package.name: set(v.references) + for v in current_vulnerabilities + } - def _is_resolved(self, previous_vuln: Vulnerability): + def is_resolved(self, vuln: Vulnerability) -> bool: """ Detects if a vulnerability has been resolved. - A vulnerability is said to be resolved when it cannot be found - in the `current_vulnerabilities`. In order to see if a vulnerability - is still present, its id and aliases are compared to values in the - `current_vulnerabilities`. It is hoped that if an ID were to change - that this would still be present in the aliases. + A vulnerability is said to be resolved when it cannot be found in + the `current_vulnerabilities`. + + Vulnerabilities are matched by the name of the affected package + and the vulnerability's "references" (set of ID and aliases). + + The vulnerability is rated as "resolved" only if there is not + intersection between previous and current references. + + This hopefully compensates in case a different ID is assigned to a + vulnerability. """ - previous_vuln_set = {previous_vuln.id, *previous_vuln.aliases} - for current_vuln in self.current_vulnerabilities: - if previous_vuln.package.name == current_vuln.package.name: - current_vuln_id_set = {current_vuln.id, *current_vuln.aliases} - if previous_vuln_set.intersection(current_vuln_id_set): - return False - return True + refs = set(vuln.references) + current = self._references.get(vuln.package.name, set()) + return not refs.intersection(current) + + +class SecurityAudit(BaseModel): + """ + Compare previous vulnerabilities to current ones and create a report + about the resolved vulnerabilities. + """ + + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + previous_vulnerabilities: list[Vulnerability] + current_vulnerabilities: list[Vulnerability] @property - def resolutions(self) -> list[Vulnerability]: + def resolved(self) -> list[Vulnerability]: """ - Return resolved vulnerabilities + Return the list of resolved vulnerabilities. """ - resolved_vulnerabilities = [] - for previous_vuln in self.previous_vulnerabilities: - if self._is_resolved(previous_vuln): - resolved_vulnerabilities.append(previous_vuln) - return resolved_vulnerabilities + matcher = VulnerabilityMatcher(self.current_vulnerabilities) + return [ + vuln for vuln in self.previous_vulnerabilities + if matcher.is_resolved(vuln) + ] + + def report_resolved_vulnerabilities(self) -> str: + if not (resolved := self.resolved): + return "" + header = cleandoc( + """ + ## Fixed Vulnerabilities + + This release fixes vulnerabilities by updating dependencies: + + | Dependency | Vulnerability | Affected | Fixed in | + |------------|---------------|----------|----------| + """ + ) + def formatted(vuln: Vulnerability) -> str: + columns = (vuln.package.name, vuln.id, str(vuln.package.version), vuln.fix_versions[0]) + return f'| {" | ".join(columns)} |' + + body = "\n".join(formatted(v) for v in resolved) + return f"{header}\n{body}" diff --git a/test/unit/util/dependencies/track_vulnerabilities_test.py b/test/unit/util/dependencies/track_vulnerabilities_test.py index 1dd584ac9..3c015c60e 100644 --- a/test/unit/util/dependencies/track_vulnerabilities_test.py +++ b/test/unit/util/dependencies/track_vulnerabilities_test.py @@ -1,55 +1,60 @@ +from exasol.toolbox.util.dependencies.audit import Vulnerability from exasol.toolbox.util.dependencies.track_vulnerabilities import ( - ResolvedVulnerabilities, + SecurityAudit, + VulnerabilityMatcher, ) -class TestResolvedVulnerabilities: - def test_vulnerability_present_for_previous_and_current(self, sample_vulnerability): +def _flip_id_and_alias(vulnerability: SampleVulnerability): + other = vulnerability + vuln_entry = { + "aliases": [other.vulnerability_id], + "id": other.cve_id, + "fix_versions": other.vulnerability.fix_versions, + "description": other.description, + } + return Vulnerability.from_audit_entry( + package_name=other.package_name, + version=other.version, + vuln_entry=vuln_entry, + ) + + +class TestVulnerabilityMatcher: + def test_not_resolved(self, sample_vulnerability): vuln = sample_vulnerability.vulnerability - resolved = ResolvedVulnerabilities( - previous_vulnerabilities=[vuln], current_vulnerabilities=[vuln] - ) - assert resolved._is_resolved(vuln) is False - - def test_vulnerability_present_for_previous_and_current_with_different_id( - self, sample_vulnerability - ): - vuln2 = sample_vulnerability.vulnerability.__dict__.copy() - vuln2["version"] = sample_vulnerability.version - # flipping aliases & id to ensure can match across types - vuln2["aliases"] = [sample_vulnerability.vulnerability_id] - vuln2["id"] = sample_vulnerability.cve_id - - resolved = ResolvedVulnerabilities( - previous_vulnerabilities=[sample_vulnerability.vulnerability], - current_vulnerabilities=[vuln2], - ) - assert resolved._is_resolved(sample_vulnerability.vulnerability) is False + matcher = VulnerabilityMatcher(current_vulnerabilities=[vuln]) + assert not matcher.is_resolved(vuln) + + def test_changed_id_not_resolved(self, sample_vulnerability): + vuln2 = _flip_id_and_alias(sample_vulnerability) + matcher = VulnerabilityMatcher(current_vulnerabilities=[vuln2]) + assert not matcher.is_resolved(sample_vulnerability.vulnerability) - def test_vulnerability_in_previous_resolved_in_current(self, sample_vulnerability): + def test_resolved(self, sample_vulnerability): vuln = sample_vulnerability.vulnerability - resolved = ResolvedVulnerabilities( - previous_vulnerabilities=[vuln], current_vulnerabilities=[] - ) - assert resolved._is_resolved(vuln) is True + matcher = VulnerabilityMatcher(current_vulnerabilities=[]) + assert matcher.is_resolved(vuln) + +class TestSecurityAudit: def test_no_vulnerabilities_for_previous_and_current(self): - resolved = ResolvedVulnerabilities( + audit = SecurityAudit( previous_vulnerabilities=[], current_vulnerabilities=[] ) - assert resolved.resolutions == [] + assert audit.resolved == [] def test_vulnerability_in_current_but_not_present(self, sample_vulnerability): - resolved = ResolvedVulnerabilities( + audit = SecurityAudit( previous_vulnerabilities=[], current_vulnerabilities=[sample_vulnerability.vulnerability], ) # only care about "resolved" vulnerabilities, not new ones - assert resolved.resolutions == [] + assert audit.resolved == [] def test_resolved_vulnerabilities(self, sample_vulnerability): - resolved = ResolvedVulnerabilities( + audit = SecurityAudit( previous_vulnerabilities=[sample_vulnerability.vulnerability], current_vulnerabilities=[], ) - assert resolved.resolutions == [sample_vulnerability.vulnerability] + assert audit.resolved == [sample_vulnerability.vulnerability] From 282591f569aba28690f80b5f5aa9d928ca8b87e2 Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 6 Apr 2026 11:25:10 +0200 Subject: [PATCH 02/21] added typehint to get_vulnerabilities_from_latest_tag --- exasol/toolbox/util/dependencies/audit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/toolbox/util/dependencies/audit.py b/exasol/toolbox/util/dependencies/audit.py index 783e21ab4..836bf8382 100644 --- a/exasol/toolbox/util/dependencies/audit.py +++ b/exasol/toolbox/util/dependencies/audit.py @@ -239,6 +239,6 @@ def get_vulnerabilities(working_directory: Path) -> list[Vulnerability]: ).vulnerabilities -def get_vulnerabilities_from_latest_tag(root_path: Path): +def get_vulnerabilities_from_latest_tag(root_path: Path) -> list[Vulnerability]: with poetry_files_from_latest_tag(root_path=root_path) as tmp_dir: return get_vulnerabilities(working_directory=tmp_dir) From 8b1e7b8d414cf968acb4fc75b17ed09ef56acace Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 6 Apr 2026 11:25:50 +0200 Subject: [PATCH 03/21] Validated warning in test and hid warning from pytest output --- test/unit/config_test.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/unit/config_test.py b/test/unit/config_test.py index 385ab2aad..035f7a6b0 100644 --- a/test/unit/config_test.py +++ b/test/unit/config_test.py @@ -1,5 +1,6 @@ from collections.abc import Iterable from pathlib import Path +from unittest.mock import Mock import pytest from pydantic_core._pydantic_core import ValidationError @@ -9,6 +10,7 @@ BaseConfig, DependencyManager, valid_version_string, + warnings, ) from exasol.toolbox.nox.plugin import hookimpl from exasol.toolbox.util.version import Version @@ -202,9 +204,19 @@ def test_raises_exception_without_hook(test_project_config_factory): class TestDependencyManager: @staticmethod - @pytest.mark.parametrize("version", ["2.1.4", "2.3.0", "2.9.9"]) - def test_works_as_expected(version): + @pytest.mark.parametrize( + "version, expected_warning", + [ + ("2.1.4", None), + ("2.3.0", None), + ("2.9.9", "Poetry version exceeds last tested version"), + ], + ) + def test_works_as_expected(version, expected_warning, monkeypatch): + monkeypatch.setattr(warnings, "warn", Mock()) DependencyManager(name="poetry", version=version) + if expected_warning: + assert expected_warning in warnings.warn.call_args.args[0] @staticmethod def test_raises_exception_when_not_supported_name(): From 91aebce639072d7448f816dfb9ded193e7bf187e Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 6 Apr 2026 11:26:28 +0200 Subject: [PATCH 04/21] Renamed method resolved to resolved_vulnerabilities --- exasol/toolbox/util/dependencies/track_vulnerabilities.py | 4 ++-- test/unit/util/dependencies/track_vulnerabilities_test.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/exasol/toolbox/util/dependencies/track_vulnerabilities.py b/exasol/toolbox/util/dependencies/track_vulnerabilities.py index 1afe5d7e0..0076969dc 100644 --- a/exasol/toolbox/util/dependencies/track_vulnerabilities.py +++ b/exasol/toolbox/util/dependencies/track_vulnerabilities.py @@ -52,7 +52,7 @@ class SecurityAudit(BaseModel): current_vulnerabilities: list[Vulnerability] @property - def resolved(self) -> list[Vulnerability]: + def resolved_vulnerabilities(self) -> list[Vulnerability]: """ Return the list of resolved vulnerabilities. """ @@ -63,7 +63,7 @@ def resolved(self) -> list[Vulnerability]: ] def report_resolved_vulnerabilities(self) -> str: - if not (resolved := self.resolved): + if not (resolved := self.resolved_vulnerabilities): return "" header = cleandoc( """ diff --git a/test/unit/util/dependencies/track_vulnerabilities_test.py b/test/unit/util/dependencies/track_vulnerabilities_test.py index 3c015c60e..33858ca56 100644 --- a/test/unit/util/dependencies/track_vulnerabilities_test.py +++ b/test/unit/util/dependencies/track_vulnerabilities_test.py @@ -42,7 +42,7 @@ def test_no_vulnerabilities_for_previous_and_current(self): audit = SecurityAudit( previous_vulnerabilities=[], current_vulnerabilities=[] ) - assert audit.resolved == [] + assert audit.resolved_vulnerabilities == [] def test_vulnerability_in_current_but_not_present(self, sample_vulnerability): audit = SecurityAudit( @@ -50,11 +50,11 @@ def test_vulnerability_in_current_but_not_present(self, sample_vulnerability): current_vulnerabilities=[sample_vulnerability.vulnerability], ) # only care about "resolved" vulnerabilities, not new ones - assert audit.resolved == [] + assert audit.resolved_vulnerabilities == [] def test_resolved_vulnerabilities(self, sample_vulnerability): audit = SecurityAudit( previous_vulnerabilities=[sample_vulnerability.vulnerability], current_vulnerabilities=[], ) - assert audit.resolved == [sample_vulnerability.vulnerability] + assert audit.resolved_vulnerabilities == [sample_vulnerability.vulnerability] From 8d097a5c62485da485f04769787a52156f921a24 Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 6 Apr 2026 11:46:24 +0200 Subject: [PATCH 05/21] Renamed nox task and class SecurityAudit once again --- exasol/toolbox/nox/_dependencies.py | 11 ++++++----- .../util/dependencies/track_vulnerabilities.py | 2 +- .../util/dependencies/track_vulnerabilities_test.py | 10 +++++----- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/exasol/toolbox/nox/_dependencies.py b/exasol/toolbox/nox/_dependencies.py index ffa0ba902..d196dfd08 100644 --- a/exasol/toolbox/nox/_dependencies.py +++ b/exasol/toolbox/nox/_dependencies.py @@ -17,7 +17,7 @@ get_licenses, ) from exasol.toolbox.util.dependencies.poetry_dependencies import get_dependencies -from exasol.toolbox.util.dependencies.track_vulnerabilities import SecurityAudit +from exasol.toolbox.util.dependencies.track_vulnerabilities import DependenciesAudit from noxconfig import PROJECT_CONFIG @@ -32,7 +32,8 @@ def dependency_licenses(session: Session) -> None: print(license_markdown.to_markdown()) -@nox.session(name="dependency:audit", python=False) +# Probably this session is obsolete +@nox.session(name="dependency:audit-old", python=False) def audit(session: Session) -> None: """Report known vulnerabilities.""" @@ -45,11 +46,11 @@ def audit(session: Session) -> None: print(json.dumps(security_issue_dict, indent=2)) -@nox.session(name="security:audit", python=False) -def security_audit(session: Session) -> None: +@nox.session(name="vulnerabilities:resolved", python=False) +def report_resolved_vulnerabilities(session: Session) -> None: """Report resolved vulnerabilities in dependencies.""" path = PROJECT_CONFIG.root_path - audit = SecurityAudit( + audit = DependenciesAudit( previous_vulnerabilities=get_vulnerabilities_from_latest_tag(path), current_vulnerabilities=get_vulnerabilities(path), ) diff --git a/exasol/toolbox/util/dependencies/track_vulnerabilities.py b/exasol/toolbox/util/dependencies/track_vulnerabilities.py index 0076969dc..ec079999c 100644 --- a/exasol/toolbox/util/dependencies/track_vulnerabilities.py +++ b/exasol/toolbox/util/dependencies/track_vulnerabilities.py @@ -40,7 +40,7 @@ def is_resolved(self, vuln: Vulnerability) -> bool: return not refs.intersection(current) -class SecurityAudit(BaseModel): +class DependenciesAudit(BaseModel): """ Compare previous vulnerabilities to current ones and create a report about the resolved vulnerabilities. diff --git a/test/unit/util/dependencies/track_vulnerabilities_test.py b/test/unit/util/dependencies/track_vulnerabilities_test.py index 33858ca56..cdfae905b 100644 --- a/test/unit/util/dependencies/track_vulnerabilities_test.py +++ b/test/unit/util/dependencies/track_vulnerabilities_test.py @@ -1,6 +1,6 @@ from exasol.toolbox.util.dependencies.audit import Vulnerability from exasol.toolbox.util.dependencies.track_vulnerabilities import ( - SecurityAudit, + DependenciesAudit, VulnerabilityMatcher, ) @@ -37,15 +37,15 @@ def test_resolved(self, sample_vulnerability): assert matcher.is_resolved(vuln) -class TestSecurityAudit: +class TestDependenciesAudit: def test_no_vulnerabilities_for_previous_and_current(self): - audit = SecurityAudit( + audit = DependenciesAudit( previous_vulnerabilities=[], current_vulnerabilities=[] ) assert audit.resolved_vulnerabilities == [] def test_vulnerability_in_current_but_not_present(self, sample_vulnerability): - audit = SecurityAudit( + audit = DependenciesAudit( previous_vulnerabilities=[], current_vulnerabilities=[sample_vulnerability.vulnerability], ) @@ -53,7 +53,7 @@ def test_vulnerability_in_current_but_not_present(self, sample_vulnerability): assert audit.resolved_vulnerabilities == [] def test_resolved_vulnerabilities(self, sample_vulnerability): - audit = SecurityAudit( + audit = DependenciesAudit( previous_vulnerabilities=[sample_vulnerability.vulnerability], current_vulnerabilities=[], ) From 9bd15c9c1663a31ad8f138de18288676174f92d9 Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 6 Apr 2026 12:03:39 +0200 Subject: [PATCH 06/21] Added integration test --- test/unit/nox/_dependencies_test.py | 33 +++++++++++++++++------------ 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/test/unit/nox/_dependencies_test.py b/test/unit/nox/_dependencies_test.py index b841d46d3..a391193e4 100644 --- a/test/unit/nox/_dependencies_test.py +++ b/test/unit/nox/_dependencies_test.py @@ -1,18 +1,25 @@ -from unittest import mock +from unittest.mock import Mock -from exasol.toolbox.nox._dependencies import audit +from exasol.toolbox.nox import _dependencies from exasol.toolbox.util.dependencies.audit import Vulnerabilities -class TestAudit: - @staticmethod - def test_works_as_expected_with_mock(nox_session, sample_vulnerability, capsys): - with mock.patch( - "exasol.toolbox.nox._dependencies.Vulnerabilities" - ) as mock_class: - mock_class.load_from_pip_audit.return_value = Vulnerabilities( - vulnerabilities=[sample_vulnerability.vulnerability] - ) - audit(nox_session) +# Proposal: Remove this test and the related nox task under test +def test_audit(monkeypatch, nox_session, sample_vulnerability, capsys): + monkeypatch.setattr(_dependencies, "Vulnerabilities", Mock()) + _dependencies.Vulnerabilities.load_from_pip_audit.return_value = Vulnerabilities( + vulnerabilities=[sample_vulnerability.vulnerability] + ) + _dependencies.audit(nox_session) + assert capsys.readouterr().out == sample_vulnerability.nox_dependencies_audit - assert capsys.readouterr().out == sample_vulnerability.nox_dependencies_audit + +def test_report_resolved_vulnerabilities(monkeypatch, nox_session, capsys, sample_vulnerability): + monkeypatch.setattr( + _dependencies, + "get_vulnerabilities_from_latest_tag", + Mock(return_value=[sample_vulnerability.vulnerability]), + ) + monkeypatch.setattr(_dependencies, "get_vulnerabilities", Mock(return_value=[])) + _dependencies.report_resolved_vulnerabilities(nox_session) + assert "| jinja2 | CVE-2025-27516 | 3.1.5 | 3.1.6 |" in capsys.readouterr().out From ebfdff7dfadd740fd1c93a489d76bbaefccc0208 Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 6 Apr 2026 17:29:18 +0200 Subject: [PATCH 07/21] Added changes file parser --- exasol/toolbox/util/release/changes_file.py | 100 ++++++++++++++++ test/unit/util/release/test_changes_file.py | 121 ++++++++++++++++++++ test/unit/util/release/test_section.py | 77 +++++++++++++ 3 files changed, 298 insertions(+) create mode 100644 exasol/toolbox/util/release/changes_file.py create mode 100644 test/unit/util/release/test_changes_file.py create mode 100644 test/unit/util/release/test_section.py diff --git a/exasol/toolbox/util/release/changes_file.py b/exasol/toolbox/util/release/changes_file.py new file mode 100644 index 000000000..b0bd23450 --- /dev/null +++ b/exasol/toolbox/util/release/changes_file.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from inspect import cleandoc + + +class ParseError(Exception): + ... + + +@dataclass +class Section: + title: str + body: str + + @property + def rendered(self) -> str: + return f"{self.title}\n\n{self.body}" + + def replace_prefix(self, prefix: str) -> None: + """ + Prepends the specified prefix to the body of the section. + + If the body starts with the first line of the specified prefix, then + replace the body's prefix. The body's prefix is all text before a + markdown list. + """ + flags = re.DOTALL | re.MULTILINE + if not self.body.startswith(prefix.splitlines()[0]): + self.body = f"{prefix}\n\n{self.body}" if self.body else prefix + elif re.search(r"^\*", self.body, flags=flags): + suffix = re.sub(r".*?^\*", "*", self.body, count=1, flags=flags) + self.body = f"{prefix}\n\n{suffix}" + else: + self.body = prefix + + +@dataclass +class ChangesFile: + """ + Represents file unreleased.md or changes_*.py in folder doc/changes/. + """ + + sections: list[Section] + + def get(self, title: str) -> Section | None: + """ + Retrieve the section with the specified title. + """ + + pattern = re.compile(f"#+ {re.escape(title)}$") + return next((s for s in self.sections if pattern.match(s.title)), None) + + def add(self, section: Section, pos: int = 1) -> None: + """ + Insert the specified section at the specified position. + """ + + self.sections.insert(pos, section) + + @property + def rendered(self) -> str: + return "\n\n".join(s.rendered for s in self.sections) + + @classmethod + def parse(cls, content: str) -> ChangesFile: + title = None + body = [] + sections = [] + + def is_body(line: str) -> bool: + return not line.startswith("#") + + def process_section(): + nonlocal sections + if title: + sections.append(Section(title, cleandoc("\n".join(body)))) + + for line in content.splitlines(): + if is_body(line): + if not title: + raise ParseError(f"Found body line without preceding title: {line}") + body.append(line) + continue + # found new title + process_section() + title = line + body = [] + + process_section() + return ChangesFile(sections) + + +def sample(): + changes = ChangesFile() + if section := changes.section(name): + section.replace_prefix(body) + else: + changes.add_section(name, body) diff --git a/test/unit/util/release/test_changes_file.py b/test/unit/util/release/test_changes_file.py new file mode 100644 index 000000000..c1ae1df48 --- /dev/null +++ b/test/unit/util/release/test_changes_file.py @@ -0,0 +1,121 @@ +from inspect import cleandoc + +import pytest + +from exasol.toolbox.util.release.changes_file import ( + ChangesFile, + ParseError, + Section, +) + +import pytest + + +class Scenario: + def __init__(self, initial: str, expected_output: str, expected_sections: list[str]): + self.testee = ChangesFile.parse(cleandoc(initial)) + self.expected_output = cleandoc(expected_output) + self.expected_sections = expected_sections + + +EMPTY = Scenario( + initial="", + expected_output="", + expected_sections=[], +) + +MINIMAL = Scenario( + initial=""" + # title + body + """, + expected_output=""" + # title + + body + """, + expected_sections=["title"], +) + +SPECIAL_CHAR_TITLE = "+ special [char] * title" + +SPECIAL_CHAR_SECTION = Scenario( + initial=f""" + # {SPECIAL_CHAR_TITLE} + body + """, + expected_output=""" + # {SPECIAL_CHAR_TITLE} + + body + """, + expected_sections=[SPECIAL_CHAR_TITLE], +) + +WITH_SUBSECTION = Scenario( + initial=""" + # title + body + ## subtitle + paragraph + + * item 1 + * item 2 + """, + expected_output=""" + # title + + body + + ## subtitle + + paragraph + + * item 1 + * item 2 + """, + expected_sections=["title","subtitle"] + ) + + +def test_parse_error() -> None: + with pytest.raises(ParseError, match="Found body line without preceding title"): + ChangesFile.parse("body line") + + +@pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, WITH_SUBSECTION]) +def test_number_of_sections(scenario: Scenario): + assert len(scenario.testee.sections) == len(scenario.expected_sections) + + +@pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, SPECIAL_CHAR_SECTION, WITH_SUBSECTION]) +def test_get(scenario: Scenario): + assert all(scenario.testee.get(s) for s in scenario.expected_sections) + + +@pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, WITH_SUBSECTION]) +def test_missing_section(scenario: Scenario): + assert scenario.testee.get("non existing") is None + + +@pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, WITH_SUBSECTION]) +def test_render(scenario: Scenario): + assert scenario.testee.rendered == scenario.expected_output + + +@pytest.fixture +def sample_section(): + return Section("# blabla", "body") + + +@pytest.mark.parametrize("scenario", [MINIMAL, WITH_SUBSECTION]) +def test_add_non_empty(scenario: Scenario, sample_section): + scenario.testee.add(sample_section) + assert scenario.testee.sections[1] == sample_section + + +@pytest.mark.parametrize("scenario", [EMPTY]) +def test_add_empty(scenario: Scenario, sample_section): + scenario.testee.add(sample_section) + assert scenario.testee.sections[0] == sample_section + diff --git a/test/unit/util/release/test_section.py b/test/unit/util/release/test_section.py new file mode 100644 index 000000000..25a580481 --- /dev/null +++ b/test/unit/util/release/test_section.py @@ -0,0 +1,77 @@ +from inspect import cleandoc + +import pytest + +from exasol.toolbox.util.release.changes_file import Section + + +class Scenario: + def __init__(self, body: str, expected_suffix: str): + self.body = cleandoc(body) + self.expected_suffix = ( + f"\n\n{cleandoc(expected_suffix)}" if expected_suffix else "" + ) + + def create_testee(self) -> Section: + return Section("# title", self.body) + + +WITHOUT_MATCHING_PREFIX = pytest.param( + Scenario("body", expected_suffix="body"), + id="without_matching_prefix", +) + +MATCHING_PREFIX_BUT_WITHOUT_LIST = pytest.param( + Scenario( + body=""" + Prefix first line + + Another line + """, + expected_suffix="", + ), + id="matching_prefix_but_without_list", +) + +MATCHING_PREFIX_AND_LIST = pytest.param( + Scenario( + """ + Prefix first line + + Another line + + * item 1 + * item 2 + """, + expected_suffix=""" + * item 1 + * item 2 + """, + ), + id="matching_prefix_and_list", +) + + +SAMPLE_PREFIX = cleandoc( + """ + Prefix first line + + | col 1 | col 2 | + |-------|-------| + | abc | 123 | + """ +) + + +@pytest.mark.parametrize( + "scenario", + [ + WITHOUT_MATCHING_PREFIX, + MATCHING_PREFIX_BUT_WITHOUT_LIST, + MATCHING_PREFIX_AND_LIST, + ], +) +def test_replace_prefix(scenario): + testee = scenario.create_testee() + testee.replace_prefix(SAMPLE_PREFIX) + assert testee.body == f"{SAMPLE_PREFIX}{scenario.expected_suffix}" From 41a85c110ceb4bb0375e1735882e1b2defbafcbd Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 7 Apr 2026 09:55:37 +0200 Subject: [PATCH 08/21] added docstring and renamed methods --- exasol/toolbox/util/release/changes_file.py | 42 ++++++++++++++++++--- test/unit/util/release/test_changes_file.py | 9 ++--- test/unit/util/release/test_section.py | 6 +-- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/exasol/toolbox/util/release/changes_file.py b/exasol/toolbox/util/release/changes_file.py index b0bd23450..bfc234ade 100644 --- a/exasol/toolbox/util/release/changes_file.py +++ b/exasol/toolbox/util/release/changes_file.py @@ -1,3 +1,30 @@ +""" +A project's Changelog is expected to list the changes coming with each of +the project's releases. The Changelog contains a file changelog.md with the +table of contents. and zero or more changes files. All files are in Markdown +syntax. + +Each changes file is named changes_*.md and describes the changes for a +specific release. The * in the file name is identical to the version number of +the related release. + +Each changes file starts with a section describing the version number, date +and name of the release and one or more subsections. The first subsection is a +summary, each other subsection lists the issues (aka. tickets) of a particular +category the are resolved by this release. Categories are security, bugfixes, +features, documentation, and refactorings. + +For the sake of simplicity, class ChangesFile maintains the sections as a +sequence, ignoring their hierarchy. + +Each section may consist of a prefix and a suffix, either might be empty. The +prefix are some introductory sentences, the suffix is the list of issues in +this section. + +Method Section.replace_prefix() adds such a prefix or replaces it, when the +section already has one. +""" + from __future__ import annotations import re @@ -6,7 +33,10 @@ class ParseError(Exception): - ... + """ + Indicates inconsistencies when parsing a changelog from raw + text. E.g. a section with a body but no title. + """ @dataclass @@ -44,7 +74,7 @@ class ChangesFile: sections: list[Section] - def get(self, title: str) -> Section | None: + def get_section(self, title: str) -> Section | None: """ Retrieve the section with the specified title. """ @@ -52,7 +82,7 @@ def get(self, title: str) -> Section | None: pattern = re.compile(f"#+ {re.escape(title)}$") return next((s for s in self.sections if pattern.match(s.title)), None) - def add(self, section: Section, pos: int = 1) -> None: + def add_section(self, section: Section, pos: int = 1) -> None: """ Insert the specified section at the specified position. """ @@ -93,8 +123,8 @@ def process_section(): def sample(): - changes = ChangesFile() - if section := changes.section(name): + changes = ChangesFile.parse(content) + if section := changes.get_section(title): section.replace_prefix(body) else: - changes.add_section(name, body) + changes.add_section(Section(title, body)) diff --git a/test/unit/util/release/test_changes_file.py b/test/unit/util/release/test_changes_file.py index c1ae1df48..2d5f80dd5 100644 --- a/test/unit/util/release/test_changes_file.py +++ b/test/unit/util/release/test_changes_file.py @@ -90,12 +90,12 @@ def test_number_of_sections(scenario: Scenario): @pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, SPECIAL_CHAR_SECTION, WITH_SUBSECTION]) def test_get(scenario: Scenario): - assert all(scenario.testee.get(s) for s in scenario.expected_sections) + assert all(scenario.testee.get_section(s) for s in scenario.expected_sections) @pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, WITH_SUBSECTION]) def test_missing_section(scenario: Scenario): - assert scenario.testee.get("non existing") is None + assert scenario.testee.get_section("non existing") is None @pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, WITH_SUBSECTION]) @@ -110,12 +110,11 @@ def sample_section(): @pytest.mark.parametrize("scenario", [MINIMAL, WITH_SUBSECTION]) def test_add_non_empty(scenario: Scenario, sample_section): - scenario.testee.add(sample_section) + scenario.testee.add_section(sample_section) assert scenario.testee.sections[1] == sample_section @pytest.mark.parametrize("scenario", [EMPTY]) def test_add_empty(scenario: Scenario, sample_section): - scenario.testee.add(sample_section) + scenario.testee.add_section(sample_section) assert scenario.testee.sections[0] == sample_section - diff --git a/test/unit/util/release/test_section.py b/test/unit/util/release/test_section.py index 25a580481..27fa2a90e 100644 --- a/test/unit/util/release/test_section.py +++ b/test/unit/util/release/test_section.py @@ -52,15 +52,13 @@ def create_testee(self) -> Section: ) -SAMPLE_PREFIX = cleandoc( - """ +SAMPLE_PREFIX = cleandoc(""" Prefix first line | col 1 | col 2 | |-------|-------| | abc | 123 | - """ -) + """) @pytest.mark.parametrize( From e8b6d82a72d3772ca3041f69c482bfd9ac270de3 Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 7 Apr 2026 10:09:59 +0200 Subject: [PATCH 09/21] Added support for list items with dash --- exasol/toolbox/util/release/changes_file.py | 4 +- test/unit/util/release/test_section.py | 71 ++++++++++++--------- 2 files changed, 42 insertions(+), 33 deletions(-) diff --git a/exasol/toolbox/util/release/changes_file.py b/exasol/toolbox/util/release/changes_file.py index bfc234ade..bcd861142 100644 --- a/exasol/toolbox/util/release/changes_file.py +++ b/exasol/toolbox/util/release/changes_file.py @@ -59,8 +59,8 @@ def replace_prefix(self, prefix: str) -> None: flags = re.DOTALL | re.MULTILINE if not self.body.startswith(prefix.splitlines()[0]): self.body = f"{prefix}\n\n{self.body}" if self.body else prefix - elif re.search(r"^\*", self.body, flags=flags): - suffix = re.sub(r".*?^\*", "*", self.body, count=1, flags=flags) + elif re.search(r"^[*-] ", self.body, flags=flags): + suffix = re.sub(r".*?^([*-])", r"\1", self.body, count=1, flags=flags) self.body = f"{prefix}\n\n{suffix}" else: self.body = prefix diff --git a/test/unit/util/release/test_section.py b/test/unit/util/release/test_section.py index 27fa2a90e..b12e9d28f 100644 --- a/test/unit/util/release/test_section.py +++ b/test/unit/util/release/test_section.py @@ -16,39 +16,46 @@ def create_testee(self) -> Section: return Section("# title", self.body) -WITHOUT_MATCHING_PREFIX = pytest.param( - Scenario("body", expected_suffix="body"), - id="without_matching_prefix", +NO_MATCHING_PREFIX = Scenario("body", expected_suffix="body") + +MATCHING_PREFIX_BUT_NO_LIST = Scenario( + body=""" + Prefix first line + + Another line + """, + expected_suffix="", ) -MATCHING_PREFIX_BUT_WITHOUT_LIST = pytest.param( - Scenario( - body=""" - Prefix first line +MATCHING_PREFIX_AND_LIST = Scenario( + body=""" + Prefix first line + + Another line - Another line - """, - expected_suffix="", - ), - id="matching_prefix_but_without_list", + * item 1 + * item 2 + """, + expected_suffix=""" + * item 1 + * item 2 + """, ) -MATCHING_PREFIX_AND_LIST = pytest.param( - Scenario( - """ - Prefix first line - - Another line - - * item 1 - * item 2 - """, - expected_suffix=""" - * item 1 - * item 2 - """, - ), - id="matching_prefix_and_list", + +LIST_WITH_DASHES = Scenario( + body=""" + Prefix first line + + Another line + + - item 1 + - item 2 + """, + expected_suffix=""" + - item 1 + - item 2 + """, ) @@ -64,12 +71,14 @@ def create_testee(self) -> Section: @pytest.mark.parametrize( "scenario", [ - WITHOUT_MATCHING_PREFIX, - MATCHING_PREFIX_BUT_WITHOUT_LIST, - MATCHING_PREFIX_AND_LIST, + pytest.param(NO_MATCHING_PREFIX, id="no_matching_prefix"), + pytest.param(MATCHING_PREFIX_BUT_NO_LIST, id="matching_prefix_but_no_list"), + pytest.param(MATCHING_PREFIX_AND_LIST, id="matching_prefix_and_list"), + pytest.param(LIST_WITH_DASHES, id="list_with_dashes"), ], ) def test_replace_prefix(scenario): testee = scenario.create_testee() testee.replace_prefix(SAMPLE_PREFIX) + expected = f"{SAMPLE_PREFIX}{scenario.expected_suffix}" assert testee.body == f"{SAMPLE_PREFIX}{scenario.expected_suffix}" From 0d6242efda4bee28e135699353b5c021f8114336 Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 7 Apr 2026 14:54:16 +0200 Subject: [PATCH 10/21] Updated docstring --- exasol/toolbox/util/release/changes_file.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/exasol/toolbox/util/release/changes_file.py b/exasol/toolbox/util/release/changes_file.py index bcd861142..f1f702142 100644 --- a/exasol/toolbox/util/release/changes_file.py +++ b/exasol/toolbox/util/release/changes_file.py @@ -17,7 +17,9 @@ For the sake of simplicity, class ChangesFile maintains the sections as a sequence, ignoring their hierarchy. -Each section may consist of a prefix and a suffix, either might be empty. The +Each section is identified by its title which should be unique. + +A section may consist of a prefix and a suffix, either might be empty. The prefix are some introductory sentences, the suffix is the list of issues in this section. @@ -53,8 +55,7 @@ def replace_prefix(self, prefix: str) -> None: Prepends the specified prefix to the body of the section. If the body starts with the first line of the specified prefix, then - replace the body's prefix. The body's prefix is all text before a - markdown list. + replace the body's prefix. """ flags = re.DOTALL | re.MULTILINE if not self.body.startswith(prefix.splitlines()[0]): From 7308052deaaee24047133448e53b9febc337f219 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 8 Apr 2026 09:22:43 +0200 Subject: [PATCH 11/21] Refactored and renamed changes_file.py Renamed to markdown.py, manage section hierarchically, manage intro and items separately, updated test cases. --- exasol/toolbox/util/release/changes_file.py | 131 ------------ exasol/toolbox/util/release/markdown.py | 156 ++++++++++++++ test/unit/util/release/test_changes_file.py | 120 ----------- test/unit/util/release/test_markdown.py | 216 ++++++++++++++++++++ test/unit/util/release/test_section.py | 84 -------- 5 files changed, 372 insertions(+), 335 deletions(-) delete mode 100644 exasol/toolbox/util/release/changes_file.py create mode 100644 exasol/toolbox/util/release/markdown.py delete mode 100644 test/unit/util/release/test_changes_file.py create mode 100644 test/unit/util/release/test_markdown.py delete mode 100644 test/unit/util/release/test_section.py diff --git a/exasol/toolbox/util/release/changes_file.py b/exasol/toolbox/util/release/changes_file.py deleted file mode 100644 index f1f702142..000000000 --- a/exasol/toolbox/util/release/changes_file.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -A project's Changelog is expected to list the changes coming with each of -the project's releases. The Changelog contains a file changelog.md with the -table of contents. and zero or more changes files. All files are in Markdown -syntax. - -Each changes file is named changes_*.md and describes the changes for a -specific release. The * in the file name is identical to the version number of -the related release. - -Each changes file starts with a section describing the version number, date -and name of the release and one or more subsections. The first subsection is a -summary, each other subsection lists the issues (aka. tickets) of a particular -category the are resolved by this release. Categories are security, bugfixes, -features, documentation, and refactorings. - -For the sake of simplicity, class ChangesFile maintains the sections as a -sequence, ignoring their hierarchy. - -Each section is identified by its title which should be unique. - -A section may consist of a prefix and a suffix, either might be empty. The -prefix are some introductory sentences, the suffix is the list of issues in -this section. - -Method Section.replace_prefix() adds such a prefix or replaces it, when the -section already has one. -""" - -from __future__ import annotations - -import re -from dataclasses import dataclass -from inspect import cleandoc - - -class ParseError(Exception): - """ - Indicates inconsistencies when parsing a changelog from raw - text. E.g. a section with a body but no title. - """ - - -@dataclass -class Section: - title: str - body: str - - @property - def rendered(self) -> str: - return f"{self.title}\n\n{self.body}" - - def replace_prefix(self, prefix: str) -> None: - """ - Prepends the specified prefix to the body of the section. - - If the body starts with the first line of the specified prefix, then - replace the body's prefix. - """ - flags = re.DOTALL | re.MULTILINE - if not self.body.startswith(prefix.splitlines()[0]): - self.body = f"{prefix}\n\n{self.body}" if self.body else prefix - elif re.search(r"^[*-] ", self.body, flags=flags): - suffix = re.sub(r".*?^([*-])", r"\1", self.body, count=1, flags=flags) - self.body = f"{prefix}\n\n{suffix}" - else: - self.body = prefix - - -@dataclass -class ChangesFile: - """ - Represents file unreleased.md or changes_*.py in folder doc/changes/. - """ - - sections: list[Section] - - def get_section(self, title: str) -> Section | None: - """ - Retrieve the section with the specified title. - """ - - pattern = re.compile(f"#+ {re.escape(title)}$") - return next((s for s in self.sections if pattern.match(s.title)), None) - - def add_section(self, section: Section, pos: int = 1) -> None: - """ - Insert the specified section at the specified position. - """ - - self.sections.insert(pos, section) - - @property - def rendered(self) -> str: - return "\n\n".join(s.rendered for s in self.sections) - - @classmethod - def parse(cls, content: str) -> ChangesFile: - title = None - body = [] - sections = [] - - def is_body(line: str) -> bool: - return not line.startswith("#") - - def process_section(): - nonlocal sections - if title: - sections.append(Section(title, cleandoc("\n".join(body)))) - - for line in content.splitlines(): - if is_body(line): - if not title: - raise ParseError(f"Found body line without preceding title: {line}") - body.append(line) - continue - # found new title - process_section() - title = line - body = [] - - process_section() - return ChangesFile(sections) - - -def sample(): - changes = ChangesFile.parse(content) - if section := changes.get_section(title): - section.replace_prefix(body) - else: - changes.add_section(Section(title, body)) diff --git a/exasol/toolbox/util/release/markdown.py b/exasol/toolbox/util/release/markdown.py new file mode 100644 index 000000000..8e5515b1d --- /dev/null +++ b/exasol/toolbox/util/release/markdown.py @@ -0,0 +1,156 @@ +""" +A project's Changelog is expected to list the changes coming with each of +the project's releases. The Changelog contains a file changelog.md with the +table of contents. and zero or more changes files. + +Each changes file is named changes_*.md and describes the changes for a +specific release. The * in the file name equals the version number of the +related release. + +All files are in Markdown syntax, divided into sections. Each section is +identified by its title which should be unique as is represented by class +Markdown. + +A section may consist of a prefix and a suffix, either might be empty. The +prefix are some introductory sentences, the suffix is the list of issues in +this section. Optionally each section can contain subsections as children. + +Method Markdown.replace_prefix() adds such a prefix or replaces it, when the +section already has one. + +The first line of each changes file must be the title describing the version +number, date and name of the release, followed by zero or multiple +sections. The first section is a summary, each other section lists the issues +(aka. tickets) of a particular category the are resolved by this +release. Categories are security, bugfixes, features, documentation, and +refactorings. +""" + +from __future__ import annotations + +import io +import re +from dataclasses import dataclass +from inspect import cleandoc + +from dataclasses import field + + +class ParseError(Exception): + """ + Indicates inconsistencies when parsing a changelog from raw + text. E.g. a section with a body but no title. + """ + + +class HierarchyError(Exception): + """ + When adding a child to a parent with higher level title. + """ + + +def is_title(line: str) -> bool: + return line and line.startswith("#") + + +def is_list_item(line: str) -> bool: + return line and (line.startswith("#") or line.startswith("-")) + + +def is_intro(line: str) -> bool: + return line and not is_title(line) and not is_list_item(line) + + +def level(title: str) -> int: + """ + Return the hierarchical level of the title, i.e. the number of "#" + chars at the beginning of the title. + """ + return len(title) - len(title.lstrip("#")) + + +@dataclass +class Markdown: + """ + Represents a Markdown file or a section within a Markdown file. + """ + + def __init__(self, title: str, intro: str, items: str, children: list[Markdown]): + self.title = title.rstrip("\n") + self.intro = intro + self.items = items + self.children = children + + def can_contain(self, child: Markdown) -> bool: + return level(self.title) < level(child.title) + + def child(self, title: str) -> Markdown | None: + """ + Retrieve the child with the specified title. + """ + + pattern = re.compile(f"#+ {re.escape(title)}$") + return next((c for c in self.children if pattern.match(c.title)), None) + + def add_child(self, child: Markdown, pos: int = 1) -> None: + """ + Insert the specified section as child at the specified position. + """ + + if not self.can_contain(child): + raise HierarchyError( + f'Markdown section "{self.title}" cannot have "{child.title}" as child.' + ) + self.children.insert(pos, child) + + @property + def rendered(self) -> str: + def elements(): + yield from (self.title, self.intro, self.items) + yield from (c.rendered for c in self.children) + + return "\n\n".join(e for e in elements() if e) + + @classmethod + def parse(cls, content: str) -> Markdown: + stream = io.StringIO(content) + line = stream.readline() + if not is_title(line): + raise ParseError( + f'First line of markdown file must be a title, but is "{line}"' + ) + + section, line = cls._parse(stream, line) + if not line: + return section + raise ParseError( + f'Found additional line "{line}" after top-level section "{section.title}".' + ) + + @classmethod + def _parse(cls, stream: io.TextIOBase, title: str) -> tuple[Markdown, str]: + intro = "" + items = "" + children = [] + + line = stream.readline() + while is_intro(line): + intro += line + line = stream.readline() + if is_list_item(line): + while not is_title(line): + items += line + line = stream.readline() + while is_title(line) and level(title) < level(line): + child, line = Markdown._parse(stream, title=line) + children.append(child) + return Markdown(title, intro.strip("\n"), items.strip("\n"), children), line + +def sample(): + changes = Markdown.parse(content) + resolved_vulnerabilities = "" + intro = resolved_vulnerabilities + if section := changes.child(title): + section.intro = intro + else: + changes.add_child(Section(title, intro, items="", [])) diff --git a/test/unit/util/release/test_changes_file.py b/test/unit/util/release/test_changes_file.py deleted file mode 100644 index 2d5f80dd5..000000000 --- a/test/unit/util/release/test_changes_file.py +++ /dev/null @@ -1,120 +0,0 @@ -from inspect import cleandoc - -import pytest - -from exasol.toolbox.util.release.changes_file import ( - ChangesFile, - ParseError, - Section, -) - -import pytest - - -class Scenario: - def __init__(self, initial: str, expected_output: str, expected_sections: list[str]): - self.testee = ChangesFile.parse(cleandoc(initial)) - self.expected_output = cleandoc(expected_output) - self.expected_sections = expected_sections - - -EMPTY = Scenario( - initial="", - expected_output="", - expected_sections=[], -) - -MINIMAL = Scenario( - initial=""" - # title - body - """, - expected_output=""" - # title - - body - """, - expected_sections=["title"], -) - -SPECIAL_CHAR_TITLE = "+ special [char] * title" - -SPECIAL_CHAR_SECTION = Scenario( - initial=f""" - # {SPECIAL_CHAR_TITLE} - body - """, - expected_output=""" - # {SPECIAL_CHAR_TITLE} - - body - """, - expected_sections=[SPECIAL_CHAR_TITLE], -) - -WITH_SUBSECTION = Scenario( - initial=""" - # title - body - ## subtitle - paragraph - - * item 1 - * item 2 - """, - expected_output=""" - # title - - body - - ## subtitle - - paragraph - - * item 1 - * item 2 - """, - expected_sections=["title","subtitle"] - ) - - -def test_parse_error() -> None: - with pytest.raises(ParseError, match="Found body line without preceding title"): - ChangesFile.parse("body line") - - -@pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, WITH_SUBSECTION]) -def test_number_of_sections(scenario: Scenario): - assert len(scenario.testee.sections) == len(scenario.expected_sections) - - -@pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, SPECIAL_CHAR_SECTION, WITH_SUBSECTION]) -def test_get(scenario: Scenario): - assert all(scenario.testee.get_section(s) for s in scenario.expected_sections) - - -@pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, WITH_SUBSECTION]) -def test_missing_section(scenario: Scenario): - assert scenario.testee.get_section("non existing") is None - - -@pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, WITH_SUBSECTION]) -def test_render(scenario: Scenario): - assert scenario.testee.rendered == scenario.expected_output - - -@pytest.fixture -def sample_section(): - return Section("# blabla", "body") - - -@pytest.mark.parametrize("scenario", [MINIMAL, WITH_SUBSECTION]) -def test_add_non_empty(scenario: Scenario, sample_section): - scenario.testee.add_section(sample_section) - assert scenario.testee.sections[1] == sample_section - - -@pytest.mark.parametrize("scenario", [EMPTY]) -def test_add_empty(scenario: Scenario, sample_section): - scenario.testee.add_section(sample_section) - assert scenario.testee.sections[0] == sample_section diff --git a/test/unit/util/release/test_markdown.py b/test/unit/util/release/test_markdown.py new file mode 100644 index 000000000..6fdca49e6 --- /dev/null +++ b/test/unit/util/release/test_markdown.py @@ -0,0 +1,216 @@ +from inspect import cleandoc + +import pytest + +from exasol.toolbox.util.release.markdown import ( + HierarchyError, + Markdown, + ParseError, +) + + +class Scenario: + def __init__( + self, initial: str, expected_output: str, expected_children: list[str] + ): + self.initial = cleandoc(initial) + self.expected_output = cleandoc(expected_output) + self.expected_children = expected_children + + def create_testee(self) -> Markdown: + return Markdown.parse(self.initial) + + +INVALID_MARKDOWN = cleandoc(""" + # Title + + Some text. + + # Another Title + """) + +MINIMAL = Scenario( + initial=""" + # title + body + """, + expected_output=""" + # title + + body + """, + expected_children=[], +) + +WITH_CHILD = Scenario( + initial=""" + # Parent + text + ## Child + paragraph + + * item 1 + * item 2 + """, + expected_output=""" + # Parent + + text + + ## Child + + paragraph + + * item 1 + * item 2 + """, + expected_children=["Child"], +) + +TWO_CHILDREN = Scenario( + initial=""" + # Parent + text + ## C1 + aaa + ## C2 + bbb + """, + expected_output=""" + # Parent + + text + + ## C1 + + aaa + + ## C2 + + bbb + """, + expected_children=["C1", "C2"], +) + + +NESTED = Scenario( + initial=""" + # Parent + text + ## Child A + aaa + ### Grand Child + ccc + ## Child B + bbb + """, + expected_output=""" + # Parent + + text + + ## Child A + + aaa + + ### Grand Child + + ccc + + ## Child B + + bbb + """, + expected_children=["Child A", "Child B"], +) + + +SPECIAL_CHAR_TITLE = "+ special [char] * title" +SPECIAL_CHAR_CHILD = Scenario( + initial=f""" + # title + + ## {SPECIAL_CHAR_TITLE} + body + """, + expected_output=f""" + # title + + ## {SPECIAL_CHAR_TITLE} + + body + """, + expected_children=[SPECIAL_CHAR_TITLE], +) + + +def test_no_title_error(): + with pytest.raises(ParseError, match="First line of markdown file must be a title"): + Markdown.parse("body\n# title") + + +def test_additional_line_error(): + expected_error = ( + 'additional line "# Another Title" after top-level section "# Title".' + ) + with pytest.raises(ParseError, match=expected_error): + Markdown.parse(INVALID_MARKDOWN) + + +ALL_SCENARIOS = [MINIMAL, WITH_CHILD, TWO_CHILDREN, NESTED, SPECIAL_CHAR_CHILD] + + +@pytest.mark.parametrize("scenario", ALL_SCENARIOS) +def test_number_of_children(scenario: Scenario): + assert len(scenario.create_testee().children) == len(scenario.expected_children) + + +@pytest.mark.parametrize("scenario", ALL_SCENARIOS) +def test_non_existing_child(scenario: Scenario): + assert scenario.create_testee().child("non existing") is None + + +@pytest.mark.parametrize("scenario", ALL_SCENARIOS) +def test_valid_child(scenario: Scenario): + assert all(scenario.create_testee().child(c) for c in scenario.expected_children) + + +@pytest.mark.parametrize("scenario", ALL_SCENARIOS) +def test_rendered(scenario: Scenario): + assert scenario.create_testee().rendered == scenario.expected_output + + +@pytest.fixture +def sample_child() -> Markdown: + return Markdown(title="## New", intro="intro", items="", children=[]) + + +@pytest.mark.parametrize( + "scenario, pos", + [ + (MINIMAL, 0), + (WITH_CHILD, 1), + (TWO_CHILDREN, 1), + ], +) +def test_add_child(sample_child: Markdown, scenario: Scenario, pos: int): + testee = scenario.create_testee() + testee.add_child(sample_child) + assert testee.children[pos] == sample_child + + +@pytest.fixture +def illegal_child() -> Markdown: + return Markdown(title="# Top-level", intro="intro", items="", children=[]) + + +@pytest.mark.parametrize("scenario", ALL_SCENARIOS) +def test_illegal_child(illegal_child: Markdown, scenario: Scenario): + testee = scenario.create_testee() + with pytest.raises(HierarchyError): + testee.add_child(illegal_child) + + +def test_nested(): + testee = NESTED.create_testee() + assert testee.child("Child A").child("Grand Child") is not None diff --git a/test/unit/util/release/test_section.py b/test/unit/util/release/test_section.py deleted file mode 100644 index b12e9d28f..000000000 --- a/test/unit/util/release/test_section.py +++ /dev/null @@ -1,84 +0,0 @@ -from inspect import cleandoc - -import pytest - -from exasol.toolbox.util.release.changes_file import Section - - -class Scenario: - def __init__(self, body: str, expected_suffix: str): - self.body = cleandoc(body) - self.expected_suffix = ( - f"\n\n{cleandoc(expected_suffix)}" if expected_suffix else "" - ) - - def create_testee(self) -> Section: - return Section("# title", self.body) - - -NO_MATCHING_PREFIX = Scenario("body", expected_suffix="body") - -MATCHING_PREFIX_BUT_NO_LIST = Scenario( - body=""" - Prefix first line - - Another line - """, - expected_suffix="", -) - -MATCHING_PREFIX_AND_LIST = Scenario( - body=""" - Prefix first line - - Another line - - * item 1 - * item 2 - """, - expected_suffix=""" - * item 1 - * item 2 - """, -) - - -LIST_WITH_DASHES = Scenario( - body=""" - Prefix first line - - Another line - - - item 1 - - item 2 - """, - expected_suffix=""" - - item 1 - - item 2 - """, -) - - -SAMPLE_PREFIX = cleandoc(""" - Prefix first line - - | col 1 | col 2 | - |-------|-------| - | abc | 123 | - """) - - -@pytest.mark.parametrize( - "scenario", - [ - pytest.param(NO_MATCHING_PREFIX, id="no_matching_prefix"), - pytest.param(MATCHING_PREFIX_BUT_NO_LIST, id="matching_prefix_but_no_list"), - pytest.param(MATCHING_PREFIX_AND_LIST, id="matching_prefix_and_list"), - pytest.param(LIST_WITH_DASHES, id="list_with_dashes"), - ], -) -def test_replace_prefix(scenario): - testee = scenario.create_testee() - testee.replace_prefix(SAMPLE_PREFIX) - expected = f"{SAMPLE_PREFIX}{scenario.expected_suffix}" - assert testee.body == f"{SAMPLE_PREFIX}{scenario.expected_suffix}" From c008cf4a2191eb052ca09fefcb6d5e87f5763c3e Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 8 Apr 2026 12:03:26 +0200 Subject: [PATCH 12/21] Enhanced class Markdown Enabled remove/replace section, added default values for constructor args, checked initial children. --- exasol/toolbox/util/release/markdown.py | 61 ++++++++++++++------ test/unit/util/release/test_markdown.py | 74 ++++++++++++++----------- 2 files changed, 88 insertions(+), 47 deletions(-) diff --git a/exasol/toolbox/util/release/markdown.py b/exasol/toolbox/util/release/markdown.py index 8e5515b1d..7496dbc73 100644 --- a/exasol/toolbox/util/release/markdown.py +++ b/exasol/toolbox/util/release/markdown.py @@ -29,11 +29,6 @@ from __future__ import annotations import io -import re -from dataclasses import dataclass -from inspect import cleandoc - -from dataclasses import field class ParseError(Exception): @@ -43,7 +38,7 @@ class ParseError(Exception): """ -class HierarchyError(Exception): +class IllegalChild(Exception): """ When adding a child to a parent with higher level title. """ @@ -69,39 +64,70 @@ def level(title: str) -> int: return len(title) - len(title.lstrip("#")) -@dataclass class Markdown: """ Represents a Markdown file or a section within a Markdown file. """ - def __init__(self, title: str, intro: str, items: str, children: list[Markdown]): + def __init__( + self, + title: str, + intro: str = "", + items: str = "", + children: list[Markdown] | None = None, + ): self.title = title.rstrip("\n") self.intro = intro self.items = items + children = children or [] + for child in children: + self._check(child) self.children = children def can_contain(self, child: Markdown) -> bool: return level(self.title) < level(child.title) + def find(self, child_title: str) -> tuple[int, Markdown] | None: + """ + Return index and child having the specified title, or None if + there is none. + """ + for i, child in enumerate(self.children): + if child.title == child_title: + return i, child + return None + def child(self, title: str) -> Markdown | None: """ Retrieve the child with the specified title. """ + return found[1] if (found := self.find(title)) else None - pattern = re.compile(f"#+ {re.escape(title)}$") - return next((c for c in self.children if pattern.match(c.title)), None) + def _check(self, child: Markdown) -> Markdown: + if not self.can_contain(child): + raise IllegalChild( + f'Markdown section "{self.title}" cannot have "{child.title}" as child.' + ) + return child def add_child(self, child: Markdown, pos: int = 1) -> None: """ Insert the specified section as child at the specified position. """ - if not self.can_contain(child): - raise HierarchyError( - f'Markdown section "{self.title}" cannot have "{child.title}" as child.' - ) - self.children.insert(pos, child) + self.children.insert(pos, self._check(child)) + + def replace_child(self, child: Markdown) -> None: + """ + If there is a child with the same title then replace this child + otherwise append the specified child. + """ + + self._check(child) + if found := self.find(child.title): + self.children[found[0]] = child + else: + self.children.append(child) @property def rendered(self) -> str: @@ -146,11 +172,14 @@ def _parse(cls, stream: io.TextIOBase, title: str) -> tuple[Markdown, str]: children.append(child) return Markdown(title, intro.strip("\n"), items.strip("\n"), children), line + def sample(): + content = "" changes = Markdown.parse(content) resolved_vulnerabilities = "" intro = resolved_vulnerabilities + title = "# title" if section := changes.child(title): section.intro = intro else: - changes.add_child(Section(title, intro, items="", [])) + changes.add_child(Markdown(title, intro)) diff --git a/test/unit/util/release/test_markdown.py b/test/unit/util/release/test_markdown.py index 6fdca49e6..24013f111 100644 --- a/test/unit/util/release/test_markdown.py +++ b/test/unit/util/release/test_markdown.py @@ -3,7 +3,7 @@ import pytest from exasol.toolbox.util.release.markdown import ( - HierarchyError, + IllegalChild, Markdown, ParseError, ) @@ -64,7 +64,7 @@ def create_testee(self) -> Markdown: * item 1 * item 2 """, - expected_children=["Child"], + expected_children=["## Child"], ) TWO_CHILDREN = Scenario( @@ -89,7 +89,7 @@ def create_testee(self) -> Markdown: bbb """, - expected_children=["C1", "C2"], + expected_children=["## C1", "## C2"], ) @@ -121,27 +121,18 @@ def create_testee(self) -> Markdown: bbb """, - expected_children=["Child A", "Child B"], + expected_children=["## Child A", "## Child B"], ) -SPECIAL_CHAR_TITLE = "+ special [char] * title" -SPECIAL_CHAR_CHILD = Scenario( - initial=f""" - # title - - ## {SPECIAL_CHAR_TITLE} - body - """, - expected_output=f""" - # title +@pytest.fixture +def sample_child() -> Markdown: + return Markdown(title="## New", intro="intro") - ## {SPECIAL_CHAR_TITLE} - body - """, - expected_children=[SPECIAL_CHAR_TITLE], -) +@pytest.fixture +def illegal_child() -> Markdown: + return Markdown(title="# Top-level", intro="intro") def test_no_title_error(): @@ -157,7 +148,12 @@ def test_additional_line_error(): Markdown.parse(INVALID_MARKDOWN) -ALL_SCENARIOS = [MINIMAL, WITH_CHILD, TWO_CHILDREN, NESTED, SPECIAL_CHAR_CHILD] +def test_constructor_illegal_child(illegal_child: Markdown): + with pytest.raises(IllegalChild): + Markdown("# title", children=[illegal_child]) + + +ALL_SCENARIOS = [MINIMAL, WITH_CHILD, TWO_CHILDREN, NESTED] @pytest.mark.parametrize("scenario", ALL_SCENARIOS) @@ -180,11 +176,6 @@ def test_rendered(scenario: Scenario): assert scenario.create_testee().rendered == scenario.expected_output -@pytest.fixture -def sample_child() -> Markdown: - return Markdown(title="## New", intro="intro", items="", children=[]) - - @pytest.mark.parametrize( "scenario, pos", [ @@ -199,18 +190,39 @@ def test_add_child(sample_child: Markdown, scenario: Scenario, pos: int): assert testee.children[pos] == sample_child -@pytest.fixture -def illegal_child() -> Markdown: - return Markdown(title="# Top-level", intro="intro", items="", children=[]) +def test_replace_illegal_child(illegal_child): + testee = WITH_CHILD.create_testee() + with pytest.raises(IllegalChild): + testee.replace_child(illegal_child) + + +@pytest.mark.parametrize("scenario", ALL_SCENARIOS) +def test_replace_existing_child(scenario: Scenario): + testee = WITH_CHILD.create_testee() + old_child = testee.children[0] + old_rendered = testee.rendered + new_child = Markdown(old_child.title, "new intro") + expected = old_rendered.replace(old_child.rendered, new_child.rendered) + testee.replace_child(new_child) + assert testee.rendered == expected + + +@pytest.mark.parametrize("scenario", ALL_SCENARIOS) +def test_replace_non_existing_child(scenario: Scenario, sample_child: Markdown): + testee = scenario.create_testee() + expected = len(testee.children) + 1 + testee.replace_child(sample_child) + assert len(testee.children) == expected + assert testee.children[-1] == sample_child @pytest.mark.parametrize("scenario", ALL_SCENARIOS) -def test_illegal_child(illegal_child: Markdown, scenario: Scenario): +def test_add_illegal_child(illegal_child: Markdown, scenario: Scenario): testee = scenario.create_testee() - with pytest.raises(HierarchyError): + with pytest.raises(IllegalChild): testee.add_child(illegal_child) def test_nested(): testee = NESTED.create_testee() - assert testee.child("Child A").child("Grand Child") is not None + assert testee.child("## Child A").child("### Grand Child") is not None From 9f7c93b024d08ad8e82aab7a220d8af0816f62b8 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 9 Apr 2026 08:20:28 +0200 Subject: [PATCH 13/21] Refactored class Markdown added methods, __eq__, read, from_string. Updated tests and docstring. --- exasol/toolbox/util/release/markdown.py | 84 +++++++++++++------------ test/unit/util/release/test_markdown.py | 28 +++++++-- 2 files changed, 65 insertions(+), 47 deletions(-) diff --git a/exasol/toolbox/util/release/markdown.py b/exasol/toolbox/util/release/markdown.py index 7496dbc73..bbf08c5cf 100644 --- a/exasol/toolbox/util/release/markdown.py +++ b/exasol/toolbox/util/release/markdown.py @@ -1,34 +1,23 @@ """ -A project's Changelog is expected to list the changes coming with each of -the project's releases. The Changelog contains a file changelog.md with the -table of contents. and zero or more changes files. - -Each changes file is named changes_*.md and describes the changes for a -specific release. The * in the file name equals the version number of the -related release. - -All files are in Markdown syntax, divided into sections. Each section is -identified by its title which should be unique as is represented by class -Markdown. - -A section may consist of a prefix and a suffix, either might be empty. The -prefix are some introductory sentences, the suffix is the list of issues in -this section. Optionally each section can contain subsections as children. - -Method Markdown.replace_prefix() adds such a prefix or replaces it, when the -section already has one. - -The first line of each changes file must be the title describing the version -number, date and name of the release, followed by zero or multiple -sections. The first section is a summary, each other section lists the issues -(aka. tickets) of a particular category the are resolved by this -release. Categories are security, bugfixes, features, documentation, and -refactorings. +Class Markdown represents a file in markdown syntax with some additional +constraints: + +* The file must start with a title in the first line. +* Each subsequent title must be of a higher level, ie. start with more "#" + characters than the top-level title. + +Each title starts a section, optionally containing an additional intro and a +bullet list of items. + +Each section can also contain subsections as children, hence sections can be +nested up to the top-level section representing the whole file. """ from __future__ import annotations import io +from pathlib import Path +from typing import Optional class ParseError(Exception): @@ -110,14 +99,15 @@ def _check(self, child: Markdown) -> Markdown: ) return child - def add_child(self, child: Markdown, pos: int = 1) -> None: + def add_child(self, child: Markdown, pos: int = 1) -> Markdown: """ Insert the specified section as child at the specified position. """ self.children.insert(pos, self._check(child)) + return self - def replace_child(self, child: Markdown) -> None: + def replace_or_append_child(self, child: Markdown) -> Markdown: """ If there is a child with the same title then replace this child otherwise append the specified child. @@ -128,6 +118,7 @@ def replace_child(self, child: Markdown) -> None: self.children[found[0]] = child else: self.children.append(child) + return self @property def rendered(self) -> str: @@ -137,9 +128,32 @@ def elements(): return "\n\n".join(e for e in elements() if e) + def __eq__(self, other) -> bool: + return isinstance(other, Markdown) and self.rendered == other.rendered + + @classmethod + def read(cls, file: Path) -> Markdown: + """ + Parse Markdown instance from the provided file. + """ + + with file.open("r") as stream: + return cls.parse(stream) + @classmethod - def parse(cls, content: str) -> Markdown: - stream = io.StringIO(content) + def from_text(cls, text: str) -> Markdown: + """ + Parse Markdown instance from the provided text. + """ + + return cls.parse(io.StringIO(text)) + + @classmethod + def parse(cls, stream: io.TextIOBase) -> Markdown: + """ + Parse Markdown instance from the provided stream. + """ + line = stream.readline() if not is_title(line): raise ParseError( @@ -171,15 +185,3 @@ def _parse(cls, stream: io.TextIOBase, title: str) -> tuple[Markdown, str]: child, line = Markdown._parse(stream, title=line) children.append(child) return Markdown(title, intro.strip("\n"), items.strip("\n"), children), line - - -def sample(): - content = "" - changes = Markdown.parse(content) - resolved_vulnerabilities = "" - intro = resolved_vulnerabilities - title = "# title" - if section := changes.child(title): - section.intro = intro - else: - changes.add_child(Markdown(title, intro)) diff --git a/test/unit/util/release/test_markdown.py b/test/unit/util/release/test_markdown.py index 24013f111..83175951c 100644 --- a/test/unit/util/release/test_markdown.py +++ b/test/unit/util/release/test_markdown.py @@ -8,6 +8,8 @@ ParseError, ) +import pytest +import pytest class Scenario: def __init__( @@ -18,7 +20,7 @@ def __init__( self.expected_children = expected_children def create_testee(self) -> Markdown: - return Markdown.parse(self.initial) + return Markdown.from_text(self.initial) INVALID_MARKDOWN = cleandoc(""" @@ -137,7 +139,7 @@ def illegal_child() -> Markdown: def test_no_title_error(): with pytest.raises(ParseError, match="First line of markdown file must be a title"): - Markdown.parse("body\n# title") + Markdown.from_text("body\n# title") def test_additional_line_error(): @@ -145,7 +147,7 @@ def test_additional_line_error(): 'additional line "# Another Title" after top-level section "# Title".' ) with pytest.raises(ParseError, match=expected_error): - Markdown.parse(INVALID_MARKDOWN) + Markdown.from_text(INVALID_MARKDOWN) def test_constructor_illegal_child(illegal_child: Markdown): @@ -153,6 +155,20 @@ def test_constructor_illegal_child(illegal_child: Markdown): Markdown("# title", children=[illegal_child]) +def test_equals() -> None: + testee = MINIMAL.create_testee() + other = MINIMAL.create_testee() + assert other == testee + other.title = "# other" + assert other != testee + + +def test_test_read(tmp_path) -> None: + file = tmp_path / "sample.md" + file.write_text(MINIMAL.initial) + assert Markdown.read(file) == MINIMAL.create_testee() + + ALL_SCENARIOS = [MINIMAL, WITH_CHILD, TWO_CHILDREN, NESTED] @@ -193,7 +209,7 @@ def test_add_child(sample_child: Markdown, scenario: Scenario, pos: int): def test_replace_illegal_child(illegal_child): testee = WITH_CHILD.create_testee() with pytest.raises(IllegalChild): - testee.replace_child(illegal_child) + testee.replace_or_append_child(illegal_child) @pytest.mark.parametrize("scenario", ALL_SCENARIOS) @@ -203,7 +219,7 @@ def test_replace_existing_child(scenario: Scenario): old_rendered = testee.rendered new_child = Markdown(old_child.title, "new intro") expected = old_rendered.replace(old_child.rendered, new_child.rendered) - testee.replace_child(new_child) + testee.replace_or_append_child(new_child) assert testee.rendered == expected @@ -211,7 +227,7 @@ def test_replace_existing_child(scenario: Scenario): def test_replace_non_existing_child(scenario: Scenario, sample_child: Markdown): testee = scenario.create_testee() expected = len(testee.children) + 1 - testee.replace_child(sample_child) + testee.replace_or_append_child(sample_child) assert len(testee.children) == expected assert testee.children[-1] == sample_child From 0f1b346eb739b8a575bfb4c681252888a366112e Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 9 Apr 2026 13:32:32 +0200 Subject: [PATCH 14/21] Fixed bug when parsing intro and items --- exasol/toolbox/util/release/markdown.py | 7 +++++-- test/unit/util/release/test_markdown.py | 11 +++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/exasol/toolbox/util/release/markdown.py b/exasol/toolbox/util/release/markdown.py index bbf08c5cf..68d40248b 100644 --- a/exasol/toolbox/util/release/markdown.py +++ b/exasol/toolbox/util/release/markdown.py @@ -38,7 +38,7 @@ def is_title(line: str) -> bool: def is_list_item(line: str) -> bool: - return line and (line.startswith("#") or line.startswith("-")) + return line and (line.startswith("*") or line.startswith("-")) def is_intro(line: str) -> bool: @@ -131,6 +131,9 @@ def elements(): def __eq__(self, other) -> bool: return isinstance(other, Markdown) and self.rendered == other.rendered + def __str__(self) -> str: + return self.rendered + @classmethod def read(cls, file: Path) -> Markdown: """ @@ -178,7 +181,7 @@ def _parse(cls, stream: io.TextIOBase, title: str) -> tuple[Markdown, str]: intro += line line = stream.readline() if is_list_item(line): - while not is_title(line): + while line and not is_title(line): items += line line = stream.readline() while is_title(line) and level(title) < level(line): diff --git a/test/unit/util/release/test_markdown.py b/test/unit/util/release/test_markdown.py index 83175951c..a6916439e 100644 --- a/test/unit/util/release/test_markdown.py +++ b/test/unit/util/release/test_markdown.py @@ -8,8 +8,6 @@ ParseError, ) -import pytest -import pytest class Scenario: def __init__( @@ -155,6 +153,15 @@ def test_constructor_illegal_child(illegal_child: Markdown): Markdown("# title", children=[illegal_child]) +def test_no_intro() -> None: + content = """ + # title + * #123: Fixed vulnerability + """ + testee = Markdown.from_text(cleandoc(content)) + assert testee == Markdown("# title", "", "* #123: Fixed vulnerability") + + def test_equals() -> None: testee = MINIMAL.create_testee() other = MINIMAL.create_testee() From fc12b43dbd6ea9e7c0379a46cf70d9489c2ee435 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 9 Apr 2026 16:46:26 +0200 Subject: [PATCH 15/21] merged changes from changelog.py --- .../features/managing_dependencies.rst | 27 +++++++++++-------- exasol/toolbox/nox/_dependencies.py | 3 +-- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/doc/user_guide/features/managing_dependencies.rst b/doc/user_guide/features/managing_dependencies.rst index 4c636e170..a89b52735 100644 --- a/doc/user_guide/features/managing_dependencies.rst +++ b/doc/user_guide/features/managing_dependencies.rst @@ -1,12 +1,17 @@ -Managing dependencies -===================== +Managing Dependencies and Vulnerabilities +========================================= -+--------------------------+------------------+----------------------------------------+ -| Nox session | CI Usage | Action | -+==========================+==================+========================================+ -| ``dependency:licenses`` | ``report.yml`` | Uses ``pip-licenses`` to return | -| | | packages with their licenses | -+--------------------------+------------------+----------------------------------------+ -| ``dependency:audit`` | No | Uses ``pip-audit`` to return active | -| | | vulnerabilities in our dependencies | -+--------------------------+------------------+----------------------------------------+ ++------------------------------+----------------+-------------------------------------+ +| Nox session | CI Usage | Action | ++==============================+================+=====================================+ +| ``dependency:licenses`` | ``report.yml`` | Uses ``pip-licenses`` to return | +| | | packages with their licenses | ++------------------------------+----------------+-------------------------------------+ +| ``dependency:audit`` | No | Uses ``pip-audit`` to report active | +| | | vulnerabilities in our dependencies | ++------------------------------+----------------+-------------------------------------+ +| ``vulnerabilities:resolved`` | No | Uses ``pip-audit`` to report known | +| | | vulnerabilities in depdendencies | +| | | that have been resolved in | +| | | comparison to the last release. | ++------------------------------+----------------+-------------------------------------+ diff --git a/exasol/toolbox/nox/_dependencies.py b/exasol/toolbox/nox/_dependencies.py index d196dfd08..c6c64401e 100644 --- a/exasol/toolbox/nox/_dependencies.py +++ b/exasol/toolbox/nox/_dependencies.py @@ -32,8 +32,7 @@ def dependency_licenses(session: Session) -> None: print(license_markdown.to_markdown()) -# Probably this session is obsolete -@nox.session(name="dependency:audit-old", python=False) +@nox.session(name="dependency:audit", python=False) def audit(session: Session) -> None: """Report known vulnerabilities.""" From f71604c751caa24a4117477d2b8826d4f976d8e5 Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 10 Apr 2026 12:36:02 +0200 Subject: [PATCH 16/21] Refactored tests --- exasol/toolbox/util/release/markdown.py | 8 +- test/unit/util/release/test_markdown.py | 174 ++++++++++++++++-------- 2 files changed, 121 insertions(+), 61 deletions(-) diff --git a/exasol/toolbox/util/release/markdown.py b/exasol/toolbox/util/release/markdown.py index 68d40248b..71da05fb3 100644 --- a/exasol/toolbox/util/release/markdown.py +++ b/exasol/toolbox/util/release/markdown.py @@ -129,7 +129,13 @@ def elements(): return "\n\n".join(e for e in elements() if e) def __eq__(self, other) -> bool: - return isinstance(other, Markdown) and self.rendered == other.rendered + return ( + isinstance(other, Markdown) and + other.title == self.title and + other.intro == self.intro and + other.items == self.items and + other.children == self.children + ) def __str__(self) -> str: return self.rendered diff --git a/test/unit/util/release/test_markdown.py b/test/unit/util/release/test_markdown.py index a6916439e..0bfbca469 100644 --- a/test/unit/util/release/test_markdown.py +++ b/test/unit/util/release/test_markdown.py @@ -9,6 +9,10 @@ ) +def _markdown(text: str) -> Markdown: + return Markdown.from_text(cleandoc(text)) + + class Scenario: def __init__( self, initial: str, expected_output: str, expected_children: list[str] @@ -21,14 +25,6 @@ def create_testee(self) -> Markdown: return Markdown.from_text(self.initial) -INVALID_MARKDOWN = cleandoc(""" - # Title - - Some text. - - # Another Title - """) - MINIMAL = Scenario( initial=""" # title @@ -42,27 +38,31 @@ def create_testee(self) -> Markdown: expected_children=[], ) -WITH_CHILD = Scenario( +FULL = Scenario( initial=""" - # Parent - text + # title + intro + * item one + * item two ## Child - paragraph - - * item 1 - * item 2 + cintro + - item c1 + - item c2 """, expected_output=""" - # Parent + # title - text + intro + + * item one + * item two ## Child - paragraph + cintro - * item 1 - * item 2 + - item c1 + - item c2 """, expected_children=["## Child"], ) @@ -124,15 +124,15 @@ def create_testee(self) -> Markdown: expected_children=["## Child A", "## Child B"], ) +CHILD = _markdown(""" + ## Sample Child + child intro. + """) -@pytest.fixture -def sample_child() -> Markdown: - return Markdown(title="## New", intro="intro") - - -@pytest.fixture -def illegal_child() -> Markdown: - return Markdown(title="# Top-level", intro="intro") +ILLEGAL_CHILD = _markdown(""" + # Top-level + intro + """) def test_no_title_error(): @@ -141,33 +141,87 @@ def test_no_title_error(): def test_additional_line_error(): + invalid_markdown = cleandoc(""" + # Title + Some text. + # Another Title + """) + expected_error = ( 'additional line "# Another Title" after top-level section "# Title".' ) with pytest.raises(ParseError, match=expected_error): - Markdown.from_text(INVALID_MARKDOWN) + Markdown.from_text(invalid_markdown) -def test_constructor_illegal_child(illegal_child: Markdown): +def test_constructor_illegal_child(): with pytest.raises(IllegalChild): - Markdown("# title", children=[illegal_child]) + Markdown("# title", children=[ILLEGAL_CHILD]) + + +@pytest.mark.parametrize("content, expected", [ + pytest.param( + """ + # title + """, + Markdown("# title"), + id="only_title", + ), + pytest.param( + """ + # title + intro + """, + Markdown("# title", "intro"), + id="intro", + ), + pytest.param( + """ + # title + * item 1 + """, + Markdown("# title", "", "* item 1"), + id="items", + ), + pytest.param( + """ + # title + intro + * item 1 + * item 2 + """, + Markdown("# title", "intro", "* item 1\n* item 2"), + id="intro_and_items", + ), + pytest.param( + """ + # title + intro + - item 1 + - item 2 + """, + Markdown("# title", "intro", "- item 1\n- item 2"), + id="intro_dash_items", + ), +]) +def test_equals(content: str, expected: Markdown) -> None: + assert Markdown.from_text(cleandoc(content)) == expected -def test_no_intro() -> None: - content = """ - # title - * #123: Fixed vulnerability - """ - testee = Markdown.from_text(cleandoc(content)) - assert testee == Markdown("# title", "", "* #123: Fixed vulnerability") - - -def test_equals() -> None: - testee = MINIMAL.create_testee() - other = MINIMAL.create_testee() - assert other == testee - other.title = "# other" - assert other != testee +@pytest.mark.parametrize( + "attr, value", + [ + ("title", "# other"), + ("intro", "other"), + ("items", "- aaa"), + ("children", []), + ], +) +def test_different(attr, value) -> None: + testee = FULL.create_testee() + other = FULL.create_testee() + setattr(other, attr, value) + assert testee != other def test_test_read(tmp_path) -> None: @@ -176,7 +230,7 @@ def test_test_read(tmp_path) -> None: assert Markdown.read(file) == MINIMAL.create_testee() -ALL_SCENARIOS = [MINIMAL, WITH_CHILD, TWO_CHILDREN, NESTED] +ALL_SCENARIOS = [MINIMAL, FULL, TWO_CHILDREN, NESTED] @pytest.mark.parametrize("scenario", ALL_SCENARIOS) @@ -203,25 +257,25 @@ def test_rendered(scenario: Scenario): "scenario, pos", [ (MINIMAL, 0), - (WITH_CHILD, 1), + (FULL, 1), (TWO_CHILDREN, 1), ], ) -def test_add_child(sample_child: Markdown, scenario: Scenario, pos: int): +def test_add_child(scenario: Scenario, pos: int): testee = scenario.create_testee() - testee.add_child(sample_child) - assert testee.children[pos] == sample_child + testee.add_child(CHILD) + assert testee.children[pos] == CHILD -def test_replace_illegal_child(illegal_child): - testee = WITH_CHILD.create_testee() +def test_replace_illegal_child(): + testee = FULL.create_testee() with pytest.raises(IllegalChild): - testee.replace_or_append_child(illegal_child) + testee.replace_or_append_child(ILLEGAL_CHILD) @pytest.mark.parametrize("scenario", ALL_SCENARIOS) def test_replace_existing_child(scenario: Scenario): - testee = WITH_CHILD.create_testee() + testee = FULL.create_testee() old_child = testee.children[0] old_rendered = testee.rendered new_child = Markdown(old_child.title, "new intro") @@ -231,19 +285,19 @@ def test_replace_existing_child(scenario: Scenario): @pytest.mark.parametrize("scenario", ALL_SCENARIOS) -def test_replace_non_existing_child(scenario: Scenario, sample_child: Markdown): +def test_replace_non_existing_child(scenario: Scenario): testee = scenario.create_testee() expected = len(testee.children) + 1 - testee.replace_or_append_child(sample_child) + testee.replace_or_append_child(CHILD) assert len(testee.children) == expected - assert testee.children[-1] == sample_child + assert testee.children[-1] == CHILD @pytest.mark.parametrize("scenario", ALL_SCENARIOS) -def test_add_illegal_child(illegal_child: Markdown, scenario: Scenario): +def test_add_illegal_child(scenario: Scenario): testee = scenario.create_testee() with pytest.raises(IllegalChild): - testee.add_child(illegal_child) + testee.add_child(ILLEGAL_CHILD) def test_nested(): From 67fa82c2916e16fe42ce99f28b4b880794148450 Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 10 Apr 2026 12:41:00 +0200 Subject: [PATCH 17/21] format:fix --- test/unit/util/release/test_markdown.py | 59 +++++++++++++------------ 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/test/unit/util/release/test_markdown.py b/test/unit/util/release/test_markdown.py index 0bfbca469..b3fba3956 100644 --- a/test/unit/util/release/test_markdown.py +++ b/test/unit/util/release/test_markdown.py @@ -146,7 +146,7 @@ def test_additional_line_error(): Some text. # Another Title """) - + expected_error = ( 'additional line "# Another Title" after top-level section "# Title".' ) @@ -159,51 +159,54 @@ def test_constructor_illegal_child(): Markdown("# title", children=[ILLEGAL_CHILD]) -@pytest.mark.parametrize("content, expected", [ - pytest.param( - """ +@pytest.mark.parametrize( + "content, expected", + [ + pytest.param( + """ # title """, - Markdown("# title"), - id="only_title", - ), - pytest.param( - """ + Markdown("# title"), + id="only_title", + ), + pytest.param( + """ # title intro """, - Markdown("# title", "intro"), - id="intro", - ), - pytest.param( - """ + Markdown("# title", "intro"), + id="intro", + ), + pytest.param( + """ # title * item 1 """, - Markdown("# title", "", "* item 1"), - id="items", - ), - pytest.param( - """ + Markdown("# title", "", "* item 1"), + id="items", + ), + pytest.param( + """ # title intro * item 1 * item 2 """, - Markdown("# title", "intro", "* item 1\n* item 2"), - id="intro_and_items", - ), - pytest.param( - """ + Markdown("# title", "intro", "* item 1\n* item 2"), + id="intro_and_items", + ), + pytest.param( + """ # title intro - item 1 - item 2 """, - Markdown("# title", "intro", "- item 1\n- item 2"), - id="intro_dash_items", - ), -]) + Markdown("# title", "intro", "- item 1\n- item 2"), + id="intro_dash_items", + ), + ], +) def test_equals(content: str, expected: Markdown) -> None: assert Markdown.from_text(cleandoc(content)) == expected From 23313668b0d4aaf7d7045ffccd788b5f072e7571 Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 10 Apr 2026 12:41:36 +0200 Subject: [PATCH 18/21] format:fix --- exasol/toolbox/util/release/markdown.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/exasol/toolbox/util/release/markdown.py b/exasol/toolbox/util/release/markdown.py index 71da05fb3..6bd9eb0c4 100644 --- a/exasol/toolbox/util/release/markdown.py +++ b/exasol/toolbox/util/release/markdown.py @@ -130,11 +130,11 @@ def elements(): def __eq__(self, other) -> bool: return ( - isinstance(other, Markdown) and - other.title == self.title and - other.intro == self.intro and - other.items == self.items and - other.children == self.children + isinstance(other, Markdown) + and other.title == self.title + and other.intro == self.intro + and other.items == self.items + and other.children == self.children ) def __str__(self) -> str: From 742ced05fa8f57a370312c3c758c64eabd46303b Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 15 Apr 2026 15:47:31 +0200 Subject: [PATCH 19/21] Updated changelog --- doc/changes/unreleased.md | 3 +++ test/unit/util/release/{test_markdown.py => markdown_test.py} | 0 2 files changed, 3 insertions(+) rename test/unit/util/release/{test_markdown.py => markdown_test.py} (100%) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 7956278c0..131f1a3db 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -2,3 +2,6 @@ ## Summary +## Refactorings + +* #763: Parse and Manipulate Changes Files diff --git a/test/unit/util/release/test_markdown.py b/test/unit/util/release/markdown_test.py similarity index 100% rename from test/unit/util/release/test_markdown.py rename to test/unit/util/release/markdown_test.py From d7c34a8b627454b857c43f4234ab8fc202fd8a5e Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 15 Apr 2026 15:58:55 +0200 Subject: [PATCH 20/21] Removed unused import --- exasol/toolbox/util/release/markdown.py | 1 - 1 file changed, 1 deletion(-) diff --git a/exasol/toolbox/util/release/markdown.py b/exasol/toolbox/util/release/markdown.py index 6bd9eb0c4..088e38e77 100644 --- a/exasol/toolbox/util/release/markdown.py +++ b/exasol/toolbox/util/release/markdown.py @@ -17,7 +17,6 @@ import io from pathlib import Path -from typing import Optional class ParseError(Exception): From dcb1003b39a2e76b1a52c14445635b7711cfb234 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 15 Apr 2026 16:01:22 +0200 Subject: [PATCH 21/21] Fixed type errors --- exasol/toolbox/util/release/markdown.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exasol/toolbox/util/release/markdown.py b/exasol/toolbox/util/release/markdown.py index 088e38e77..ffd6daf55 100644 --- a/exasol/toolbox/util/release/markdown.py +++ b/exasol/toolbox/util/release/markdown.py @@ -33,15 +33,15 @@ class IllegalChild(Exception): def is_title(line: str) -> bool: - return line and line.startswith("#") + return bool(line) and line.startswith("#") def is_list_item(line: str) -> bool: - return line and (line.startswith("*") or line.startswith("-")) + return bool(line) and (line.startswith("*") or line.startswith("-")) def is_intro(line: str) -> bool: - return line and not is_title(line) and not is_list_item(line) + return bool(line) and not is_title(line) and not is_list_item(line) def level(title: str) -> int: