Add severity-aware exit codes to traffic_ctl

doc/appendices/command-line/traffic_ctl.en.rst

@@ -114,6 +114,48 @@ Options
 
    Path to the runroot file.
 
+.. option:: -e, --error-level <level>
+
+   Set the minimum severity that causes a non-zero exit code. When the server returns an error
+   response with annotated data entries, ``traffic_ctl`` inspects the ``severity`` of each
+   annotation and compares the highest severity against this threshold. If the highest severity
+   is **at or above** the threshold, the process exits with code ``2``; otherwise it exits ``0``.
+
+   Accepted values (case-insensitive): ``diag``, ``debug``, ``status``, ``note``, ``warn``,
+   ``error``, ``fatal``, ``alert``, ``emergency``.
+
+   Default: ``error``.
+
+   This option only affects error responses that contain annotated data entries (i.e., entries
+   in the ``data`` array with a ``severity`` field).
+
+   .. note::
+
+      Protocol-level JSONRPC errors (e.g., ``-32601 Method not found``, ``-32600 Invalid
+      Request``) and application errors with no ``data`` entries **always exit** ``2`` regardless
+      of ``--error-level``. These errors indicate a communication or dispatch failure — not an
+      application-level condition — and are therefore unconditionally treated as hard errors.
+
+   Examples:
+
+   .. code-block:: bash
+
+      # Default: annotations without explicit severity default to DIAG.
+      # DIAG < ERROR, so exit 0.
+      $ traffic_ctl server drain   # "already draining" is DIAG
+      $ echo $?
+      0
+
+      # Treat any annotation (including DIAG) as an error
+      $ traffic_ctl --error-level=diag server drain
+      $ echo $?
+      2
+
+      # Only fail on fatal or above
+      $ traffic_ctl --error-level=fatal server drain
+      $ echo $?
+      0
+
 Subcommands
 ===========
 
@@ -1299,13 +1341,18 @@ Exit Codes
 :program:`traffic_ctl` uses the following exit codes:
 
 ``0``
-   Success. The requested operation completed successfully.
+   Success. The requested operation completed successfully, or the server returned an error
+   whose annotations all have severity below the ``--error-level`` threshold (default: ``error``).
 
 ``2``
    Error. The operation failed. This may be returned when:
 
    - The RPC communication with :program:`traffic_server` failed (e.g. socket not found or connection refused).
-   - The server response contains an error (e.g. invalid record name, malformed request).
+   - The server response is a protocol-level JSONRPC error (e.g. ``-32601 Method not found``) or
+     an application error with no ``data`` entries. These **always** exit ``2`` regardless of
+     ``--error-level``.
+   - The server response contains data annotations whose highest severity is at or above the
+     ``--error-level`` threshold.
 
 ``3``
    Unimplemented. The requested command is not yet implemented.

doc/developer-guide/jsonrpc/jsonrpc-node-errors.en.rst

@@ -49,7 +49,8 @@ different set of errors in the following format:
       "jsonrpc": "2.0"
    }
 
-In some cases the data field could be populated:
+In some cases the data field could be populated. Each entry in the ``data`` array contains a
+``code``, a ``severity``, and a ``message``:
 
 .. code-block:: json
 
@@ -61,13 +62,35 @@ In some cases the data field could be populated:
          "data":[
             {
                "code": 2,
+               "severity": 5,
                "message":"Denied privileged API access for uid=XXX gid=XXX"
             }
          ]
       },
       "id":"5e273ec0-3e3b-4a81-90ec-aeee3d38073f"
    }
 
+The ``severity`` field is an integer that corresponds to the ``swoc::Errata::Severity`` levels.
+It is always present in the response. If the handler does not set an explicit severity for an
+annotation, the server defaults to ``0`` (Diag).
+
+====  ===========
+Code  Severity
+====  ===========
+0     Diag
+1     Debug
+2     Status
+3     Note
+4     Warn
+5     Error
+6     Fatal
+7     Alert
+8     Emergency
+====  ===========
+
+The ``severity`` field is used by :program:`traffic_ctl` to determine the exit code via the
+``--error-level`` option. See :ref:`traffic_ctl_jsonrpc` for details.
+
 
 .. _jsonrpc-node-errors-standard-errors:
 
@@ -128,6 +151,7 @@ Under this error, the `data` field could be populated with the following errors,
    "data":[
       {
          "code":2,
+         "severity": 5,
          "message":"Denied privileged API access for uid=XXX gid=XXX"
       }
    ]

include/mgmt/rpc/jsonrpc/json/YAMLCodec.h

@@ -195,6 +195,9 @@ class yamlcpp_json_encoder
       for (auto const &err : errata) {
         json << YAML::BeginMap;
         json << YAML::Key << "code" << YAML::Value << errata.code().value();
+        // Default to DIAG for annotations without explicit severity — this is a wire format
+        // concern, not stored in the Errata model nor in the RPC error model.
+        json << YAML::Key << "severity" << YAML::Value << static_cast<int>(err.has_severity() ? err.severity() : ERRATA_DIAG);
         json << YAML::Key << "message" << YAML::Value << std::string{err.text().data(), err.text().size()};
         json << YAML::EndMap;
       }

include/shared/rpc/RPCRequests.h

@@ -22,6 +22,7 @@
 #include <string>
 #include <variant>
 #include "tsutil/ts_bw_format.h"
+#include "tsutil/ts_errata.h"
 #include <yaml-cpp/yaml.h>
 #include <tscore/ink_uuid.h>
 
@@ -68,9 +69,13 @@ struct JSONRPCResponse {
 struct JSONRPCError {
   int32_t     code;    //!< High level error code.
   std::string message; //!< High level message
-  // the following data is defined by TS, it will be a key/value pair.
-  std::vector<std::pair<int32_t, std::string>> data;
-  friend std::ostream                         &operator<<(std::ostream &os, const JSONRPCError &err);
+  struct DataEntry {
+    int32_t     code;
+    int32_t     severity{0}; //!< Per-annotation severity, defaults to DIAG (0).
+    std::string message;
+  };
+  std::vector<DataEntry> data;
+  friend std::ostream   &operator<<(std::ostream &os, const JSONRPCError &err);
 };
 
 /**
@@ -206,9 +211,16 @@ inline std::ostream &
 operator<<(std::ostream &os, const JSONRPCError &err)
 {
   os << "Server Error found:\n";
-  os << "[" << err.code << "] " << err.message << '\n';
-  for (auto &&[code, message] : err.data) {
-    os << "- [" << code << "] " << message << '\n';
+  os << "[" << err.code << "] " << err.message;
+  if (!err.data.empty()) {
+    os << " [code: " << err.data[0].code << "]";
+  }
+  os << '\n';
+
+  for (auto const &entry : err.data) {
+    auto           sev  = static_cast<size_t>(entry.severity);
+    swoc::TextView name = sev < Severity_Names.size() ? Severity_Names[sev] : "Unknown";
+    os << "- " << name << ": " << entry.message << '\n';
   }
 
   return os;

include/shared/rpc/yaml_codecs.h

@@ -65,7 +65,11 @@ template <> struct convert<shared::rpc::JSONRPCError> {
     error.message = helper::try_extract<std::string>(node, "message");
     if (auto data = node["data"]) {
       for (auto &&err : data) {
-        error.data.emplace_back(helper::try_extract<int32_t>(err, "code"), helper::try_extract<std::string>(err, "message"));
+        shared::rpc::JSONRPCError::DataEntry entry;
+        entry.code     = helper::try_extract<int32_t>(err, "code");
+        entry.severity = helper::try_extract<int32_t>(err, "severity", false, 0);
+        entry.message  = helper::try_extract<std::string>(err, "message");
+        error.data.push_back(std::move(entry));
       }
     }
     return true;

src/mgmt/rpc/jsonrpc/unit_tests/test_basic_protocol.cc

@@ -21,6 +21,7 @@
 #include <catch2/catch_test_macros.hpp> /* catch unit-test framework */
 
 #include <tsutil/ts_bw_format.h>
+#include <tsutil/ts_errata.h>
 
 #include "mgmt/rpc/jsonrpc/JsonRPCManager.h"
 #include "mgmt/rpc/jsonrpc/JsonRPC.h"
@@ -79,6 +80,14 @@ test_callback_ok_or_error(std::string_view const & /* id ATS_UNUSED */, YAML::No
   return resp;
 }
 
+inline swoc::Rv<YAML::Node>
+test_callback_with_severity(std::string_view const & /* id ATS_UNUSED */, YAML::Node const & /* params ATS_UNUSED */)
+{
+  swoc::Rv<YAML::Node> resp;
+  resp.errata().assign(ERR1).note(ERRATA_WARN, "this is a warning").note("this has no severity");
+  return resp;
+}
+
 static int notificationCallCount{0};
 inline void
 test_nofitication(YAML::Node const & /* params ATS_UNUSED */)
@@ -140,7 +149,7 @@ TEST_CASE("Register/call method - respond with errors (data field)", "[method][e
         R"({"jsonrpc": "2.0", "method": "test_callback_ok_or_error", "params": {"return_error": "yes"}, "id": "14"})");
       REQUIRE(json);
       const std::string_view expected =
-        R"({"jsonrpc": "2.0", "error": {"code": 9, "message": "Error during execution", "data": [{"code": 9999, "message": "Just an error message to add more meaning to the failure"}]}, "id": "14"})";
+        R"({"jsonrpc": "2.0", "error": {"code": 9, "message": "Error during execution", "data": [{"code": 9999, "severity": 0, "message": "Just an error message to add more meaning to the failure"}]}, "id": "14"})";
       REQUIRE(*json == expected);
     }
   }
@@ -184,7 +193,7 @@ TEST_CASE("Basic test, batch calls", "[methods][notifications]")
 
       REQUIRE(resp1);
       const std::string_view expected =
-        R"([{"jsonrpc": "2.0", "result": {"ran": "ok"}, "id": "13"}, {"jsonrpc": "2.0", "error": {"code": 9, "message": "Error during execution", "data": [{"code": 9999, "message": "Just an error message to add more meaning to the failure"}]}, "id": "14"}])";
+        R"([{"jsonrpc": "2.0", "result": {"ran": "ok"}, "id": "13"}, {"jsonrpc": "2.0", "error": {"code": 9, "message": "Error during execution", "data": [{"code": 9999, "severity": 0, "message": "Just an error message to add more meaning to the failure"}]}, "id": "14"}])";
       REQUIRE(*resp1 == expected);
     }
   }
@@ -608,3 +617,19 @@ TEST_CASE("Call method with invalid ID", "[invalid_id]")
     REQUIRE(*resp == expected);
   }
 }
+
+TEST_CASE("Severity field in error data entries", "[severity]")
+{
+  JsonRpcUnitTest rpc;
+
+  SECTION("Annotation with severity emits it, annotation without defaults to DIAG (0)")
+  {
+    REQUIRE(rpc.add_method_handler("test_callback_with_severity", &test_callback_with_severity));
+
+    const auto json = rpc.handle_call(R"({"jsonrpc": "2.0", "method": "test_callback_with_severity", "params": {}, "id": "50"})");
+    REQUIRE(json);
+    const std::string_view expected =
+      R"({"jsonrpc": "2.0", "error": {"code": 9, "message": "Error during execution", "data": [{"code": 9999, "severity": 4, "message": "this is a warning"}, {"code": 9999, "severity": 0, "message": "this has no severity"}]}, "id": "50"})";
+    REQUIRE(*json == expected);
+  }
+}

src/traffic_ctl/CMakeLists.txt

@@ -32,3 +32,12 @@ target_link_libraries(traffic_ctl ts::tscore ts::config libswoc::libswoc yaml-cp
 install(TARGETS traffic_ctl)
 
 clang_tidy_check(traffic_ctl)
+
+if(BUILD_TESTING)
+  add_executable(test_traffic_ctl_status unit_tests/test_traffic_ctl_status.cc)
+  target_include_directories(test_traffic_ctl_status PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+  target_link_libraries(
+    test_traffic_ctl_status Catch2::Catch2WithMain ts::tscore ts::tsutil libswoc::libswoc yaml-cpp::yaml-cpp
+  )
+  add_catch2_test(NAME test_traffic_ctl_status COMMAND test_traffic_ctl_status)
+endif()

src/traffic_ctl/CtrlPrinters.cc

@@ -67,7 +67,7 @@ BasePrinter::write_output(shared::rpc::JSONRPCResponse const &response)
   }
 
   if (response.is_error()) {
-    App_Exit_Status_Code = CTRL_EX_ERROR; // Set the exit code to error, so we can return it later.
+    App_Exit_Status_Code = appExitCodeFromResponse(response);
 
     // If an error is present, then as per the specs we can ignore the jsonrpc.result field,
     // so we print the error and we are done here!

src/traffic_ctl/TrafficCtlStatus.h

@@ -20,10 +20,37 @@ limitations under the License.
 */
 #pragma once
 
+#include <algorithm>
+
+#include "shared/rpc/RPCRequests.h"
+#include "shared/rpc/yaml_codecs.h"
+#include "tsutil/ts_errata.h"
+
 constexpr int CTRL_EX_OK = 0;
 // EXIT_FAILURE can also be used.
 constexpr int CTRL_EX_ERROR         = 2;
 constexpr int CTRL_EX_UNIMPLEMENTED = 3;
 constexpr int CTRL_EX_TEMPFAIL = 75; ///< Temporary failure — operation in progress, retry later (EX_TEMPFAIL from sysexits.h).
 
-extern int App_Exit_Status_Code; //!< Global variable to store the exit status code of the application.
+extern int                         App_Exit_Status_Code; //!< Global variable to store the exit status code of the application.
+extern swoc::Errata::severity_type App_Exit_Level_Error; //!< Minimum severity to treat as error for exit status.
+
+inline int
+appExitCodeFromResponse(const shared::rpc::JSONRPCResponse &response)
+{
+  if (!response.is_error()) {
+    return CTRL_EX_OK;
+  }
+
+  auto err = response.error.as<shared::rpc::JSONRPCError>();
+  if (err.data.empty()) {
+    return CTRL_EX_ERROR;
+  }
+
+  auto effective_severity = [](auto const &e) { return swoc::Errata::severity_type(e.severity); };
+
+  auto it = std::max_element(err.data.begin(), err.data.end(),
+                             [&](auto const &a, auto const &b) { return effective_severity(a) < effective_severity(b); });
+
+  return effective_severity(*it) >= App_Exit_Level_Error ? CTRL_EX_ERROR : CTRL_EX_OK;
+}

src/traffic_ctl/traffic_ctl.cc

@@ -21,6 +21,7 @@
   limitations under the License.
  */
 
+#include <algorithm>
 #include <iostream>
 #include <csignal>
 
@@ -35,9 +36,10 @@
 #include "FileConfigCommand.h"
 #include "SSLMultiCertCommand.h"
 #include "TrafficCtlStatus.h"
+#include "tsutil/ts_errata.h"
 
-// Define the global variable
-int App_Exit_Status_Code = CTRL_EX_OK; // Initialize it to a default value
+int                         App_Exit_Status_Code = CTRL_EX_OK;
+swoc::Errata::severity_type App_Exit_Level_Error = ERRATA_ERROR;
 namespace
 {
 void
@@ -91,7 +93,10 @@ main([[maybe_unused]] int argc, const char **argv)
     .add_option("--format", "-f", "Use a specific output format {json|rpc}", "", 1, "", "format")
     .add_option("--read-timeout-ms", "", "Read timeout for RPC (in milliseconds)", "", 1, "10000", "read-timeout")
     .add_option("--read-attempts", "", "Read attempts for RPC", "", 1, "100", "read-attempts")
-    .add_option("--watch", "-w", "Execute a program periodically. Watch interval(in seconds) can be passed.", "", 1, "-1", "watch");
+    .add_option("--watch", "-w", "Execute a program periodically. Watch interval(in seconds) can be passed.", "", 1, "-1", "watch")
+    .add_option("--error-level", "-e",
+                "Minimum severity to treat as error for exit status {diag|debug|status|note|warn|error|fatal|alert|emergency}", "",
+                1, "error", "error-level");
 
   auto &config_command     = parser.add_command("config", "Manipulate configuration records").require_commands();
   auto &metric_command     = parser.add_command("metric", "Manipulate performance metrics").require_commands();
@@ -338,6 +343,15 @@ main([[maybe_unused]] int argc, const char **argv)
     signal_register_handler(SIGINT, handle_signal);
 
     auto args = parser.parse(argv);
+
+    auto error_level_str = args.get("error-level").value();
+    auto it =
+      std::find_if(Severity_Names.begin(), Severity_Names.end(), [&](auto name) { return strcasecmp(name, error_level_str) == 0; });
+    if (it == Severity_Names.end()) {
+      throw std::runtime_error(std::string("Unknown error level: ") + std::string(error_level_str));
+    }
+    App_Exit_Level_Error = swoc::Errata::severity_type(std::distance(Severity_Names.begin(), it));
+
     argparser_runroot_handler(args.get("run-root").value(), argv[0]);
     Layout::create();