diff --git a/datadog_sync/commands/shared/options.py b/datadog_sync/commands/shared/options.py index 68869aa4..ce271a34 100644 --- a/datadog_sync/commands/shared/options.py +++ b/datadog_sync/commands/shared/options.py @@ -231,6 +231,17 @@ def click_config_file_provider(ctx: Context, opts: CustomOptionClass, value: Non "will create a resource file for each individual resource.", cls=CustomOptionClass, ), + option( + "--json", + "emit_json", + envvar=constants.DD_SYNC_JSON, + required=False, + is_flag=True, + default=False, + show_default=True, + help="Emit resource-level outcomes as JSON lines to stdout.", + cls=CustomOptionClass, + ), ] _storage_options = [ diff --git a/datadog_sync/constants.py b/datadog_sync/constants.py index 0ca54ec0..90297938 100644 --- a/datadog_sync/constants.py +++ b/datadog_sync/constants.py @@ -26,6 +26,7 @@ DD_SHOW_PROGRESS_BAR = "DD_SHOW_PROGRESS_BAR" DD_VERIFY_SSL_CERTIFICATES = "DD_VERIFY_SSL_CERTIFICATES" DD_ALLOW_PARTIAL_PERMISSIONS_ROLES = "DD_ALLOW_PARTIAL_PERMISSIONS_ROLES" +DD_SYNC_JSON = "DD_SYNC_JSON" LOCAL_STORAGE_TYPE = "local" S3_STORAGE_TYPE = "s3" diff --git a/datadog_sync/utils/configuration.py b/datadog_sync/utils/configuration.py index b8f87260..21272804 100644 --- a/datadog_sync/utils/configuration.py +++ b/datadog_sync/utils/configuration.py @@ -64,6 +64,7 @@ class Configuration(object): backup_before_reset: bool show_progress_bar: bool allow_self_lockout: bool + emit_json: bool = False allow_partial_permissions_roles: List[str] = field(default_factory=list) resources: Dict[str, BaseResource] = field(default_factory=dict) resources_arg: List[str] = field(default_factory=list) @@ -318,6 +319,7 @@ def build_config(cmd: Command, **kwargs: Optional[Any]) -> Configuration: backup_before_reset=backup_before_reset, show_progress_bar=show_progress_bar, allow_self_lockout=allow_self_lockout, + emit_json=kwargs.get("emit_json", False), allow_partial_permissions_roles=allow_partial_permissions_roles, ) diff --git a/datadog_sync/utils/sync_report.py b/datadog_sync/utils/sync_report.py new file mode 100644 index 00000000..59e8b363 --- /dev/null +++ b/datadog_sync/utils/sync_report.py @@ -0,0 +1,64 @@ +# Unless explicitly stated otherwise all files in this repository are licensed +# under the 3-clause BSD style license (see LICENSE). +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019 Datadog, Inc. + +from __future__ import annotations + +import json +import sys +from dataclasses import asdict, dataclass +from typing import Dict, Literal + + +_REASON_MAX_LEN = 1024 + + +@dataclass +class ResourceOutcome: + """A single resource-level outcome emitted as a JSON line to stdout. + + Field names and values are aligned with the CLI's ``datadog.org-sync.action`` + metric tags so that downstream consumers (e.g. the managed-sync Go worker) can + forward them as statsd tags without translation. + + Metric-tag mapping:: + + JSON field CLI metric tag Values + ───────────── ────────────── ────── + resource_type resource_type:X dashboards, monitors, ... + id id:X resource identifier (empty for type-level failures) + action_type action_type:X import | sync | delete + status status:X success | skipped | failure | filtered + action_sub_type action_sub_type:X create | update | "" (sync only) + reason reason:X freetext explanation (truncated to 1024 chars) + + Note: ``filtered`` is a JSON-only status. The CLI metric (``datadog.org-sync.action``) + is not emitted for filtered resources, so this value has no metric-tag counterpart. + + Diffs-mode semantics: In ``diffs`` mode, outcomes describe *intended* actions, not + completed mutations. A ``status:success`` with ``action_type:delete`` means "this + resource would be deleted", not "this resource was deleted". Consumers that distinguish + dry-run from live should check which CLI command was invoked. + + Stdout/stderr contract: JSON outcomes go to stdout; all logging and progress output + goes to stderr. Machine consumers should pipe stdout only. + """ + + resource_type: str + id: str + action_type: Literal["import", "sync", "delete"] + status: Literal["success", "skipped", "failure", "filtered"] + action_sub_type: Literal["create", "update", ""] # only populated on sync success + reason: str # empty for success, explanation for skip/fail + + def __post_init__(self) -> None: + if len(self.reason) > _REASON_MAX_LEN: + self.reason = self.reason[:_REASON_MAX_LEN] + "...(truncated)" + + def to_dict(self) -> Dict[str, str]: + return asdict(self) + + def emit(self) -> None: + """Write this outcome as a single JSON line to stdout.""" + print(json.dumps(self.to_dict()), file=sys.stdout, flush=True) diff --git a/tests/unit/test_sync_report.py b/tests/unit/test_sync_report.py new file mode 100644 index 00000000..1dbdad74 --- /dev/null +++ b/tests/unit/test_sync_report.py @@ -0,0 +1,164 @@ +# Unless explicitly stated otherwise all files in this repository are licensed +# under the 3-clause BSD style license (see LICENSE). +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019 Datadog, Inc. + +import json +from io import StringIO +from unittest.mock import patch + +from datadog_sync.utils.sync_report import ResourceOutcome, _REASON_MAX_LEN + + +class TestResourceOutcome: + def test_create_outcome(self): + outcome = ResourceOutcome( + resource_type="dashboards", + id="abc-123", + action_type="sync", + status="success", + action_sub_type="create", + reason="", + ) + assert outcome.resource_type == "dashboards" + assert outcome.id == "abc-123" + assert outcome.action_type == "sync" + assert outcome.status == "success" + assert outcome.action_sub_type == "create" + assert outcome.reason == "" + + def test_create_outcome_with_reason(self): + outcome = ResourceOutcome( + resource_type="monitors", + id="12345", + action_type="import", + status="skipped", + action_sub_type="", + reason="Synthetics monitors are created by synthetics tests.", + ) + assert outcome.status == "skipped" + assert outcome.reason == "Synthetics monitors are created by synthetics tests." + + def test_outcome_to_dict(self): + outcome = ResourceOutcome( + resource_type="dashboards", + id="abc-123", + action_type="sync", + status="failure", + action_sub_type="", + reason="500 Internal Server Error", + ) + d = outcome.to_dict() + assert d == { + "resource_type": "dashboards", + "id": "abc-123", + "action_type": "sync", + "status": "failure", + "action_sub_type": "", + "reason": "500 Internal Server Error", + } + + +class TestEmit: + def test_emit_writes_json_line_to_stdout(self): + outcome = ResourceOutcome("dashboards", "abc-123", "sync", "success", "create", "") + buf = StringIO() + with patch("sys.stdout", buf): + outcome.emit() + + parsed = json.loads(buf.getvalue().strip()) + assert parsed["resource_type"] == "dashboards" + assert parsed["id"] == "abc-123" + assert parsed["action_type"] == "sync" + assert parsed["status"] == "success" + assert parsed["action_sub_type"] == "create" + assert parsed["reason"] == "" + + def test_emit_one_line_per_call(self): + """Each emit() produces exactly one line.""" + buf = StringIO() + with patch("sys.stdout", buf): + ResourceOutcome("dashboards", "a", "sync", "success", "create", "").emit() + ResourceOutcome("monitors", "1", "sync", "failure", "", "err").emit() + + lines = buf.getvalue().strip().split("\n") + assert len(lines) == 2 + + def test_emit_is_valid_jsonl(self): + """Multiple emits produce valid JSONL (each line is independent JSON).""" + buf = StringIO() + with patch("sys.stdout", buf): + ResourceOutcome("dashboards", "a", "sync", "success", "create", "").emit() + ResourceOutcome("monitors", "1", "import", "skipped", "", "synth alert").emit() + ResourceOutcome("monitors", "2", "sync", "failure", "", "500").emit() + + lines = buf.getvalue().strip().split("\n") + for line in lines: + parsed = json.loads(line) + assert "resource_type" in parsed + assert "status" in parsed + + def test_emit_includes_reason(self): + buf = StringIO() + with patch("sys.stdout", buf): + ResourceOutcome("monitors", "123", "sync", "failure", "", "403 Forbidden").emit() + + parsed = json.loads(buf.getvalue().strip()) + assert parsed["reason"] == "403 Forbidden" + + def test_emit_all_statuses(self): + """All 4 status values emit valid JSON.""" + statuses = ["success", "skipped", "failure", "filtered"] + buf = StringIO() + with patch("sys.stdout", buf): + for status in statuses: + ResourceOutcome("dashboards", "x", "sync", status, "", "").emit() + + lines = buf.getvalue().strip().split("\n") + assert len(lines) == 4 + emitted_statuses = {json.loads(line)["status"] for line in lines} + assert emitted_statuses == set(statuses) + + def test_emit_all_action_types(self): + """All 3 action_type values emit valid JSON.""" + action_types = ["import", "sync", "delete"] + buf = StringIO() + with patch("sys.stdout", buf): + for action_type in action_types: + ResourceOutcome("dashboards", "x", action_type, "success", "", "").emit() + + lines = buf.getvalue().strip().split("\n") + emitted_action_types = {json.loads(line)["action_type"] for line in lines} + assert emitted_action_types == set(action_types) + + def test_emit_action_sub_types(self): + """action_sub_type values emit correctly.""" + buf = StringIO() + with patch("sys.stdout", buf): + ResourceOutcome("dashboards", "a", "sync", "success", "create", "").emit() + ResourceOutcome("dashboards", "b", "sync", "success", "update", "").emit() + ResourceOutcome("dashboards", "c", "import", "success", "", "").emit() + + lines = buf.getvalue().strip().split("\n") + assert json.loads(lines[0])["action_sub_type"] == "create" + assert json.loads(lines[1])["action_sub_type"] == "update" + assert json.loads(lines[2])["action_sub_type"] == "" + + +class TestReasonTruncation: + def test_short_reason_unchanged(self): + outcome = ResourceOutcome("dashboards", "a", "sync", "failure", "", "short error") + assert outcome.reason == "short error" + + def test_long_reason_truncated(self): + long_reason = "x" * 2000 + outcome = ResourceOutcome("dashboards", "a", "sync", "failure", "", long_reason) + assert len(outcome.reason) == _REASON_MAX_LEN + len("...(truncated)") + assert outcome.reason.endswith("...(truncated)") + assert outcome.reason.startswith("x" * _REASON_MAX_LEN) + + def test_exact_limit_not_truncated(self): + exact_reason = "y" * _REASON_MAX_LEN + outcome = ResourceOutcome("dashboards", "a", "sync", "failure", "", exact_reason) + assert outcome.reason == exact_reason +