bench: maintenance

bench/locli/src/Cardano/Unlog/BackendDB.hs

@@ -17,7 +17,7 @@ module Cardano.Unlog.BackendDB
        ) where
 
 import           Cardano.Analysis.API.Ground (Host (..), LogObjectSource (..))
-import           Cardano.Prelude (ExceptT)
+import           Cardano.Prelude (ExceptT, (<|>))
 import           Cardano.Unlog.LogObject (HostLogs (..), LogObject (..), RunLogs (..), fromTextRef)
 import           Cardano.Unlog.LogObjectDB
 import           Cardano.Util (sequenceConcurrentlyChunksOf, withTimingInfo)
@@ -35,6 +35,7 @@ import qualified Data.Map.Strict as Map
 import           Data.Maybe
 import qualified Data.Text.Short as ShortText (unpack)
 import           Data.Time.Clock (UTCTime, getCurrentTime)
+import           Data.Time.Format.ISO8601
 import           GHC.Conc (numCapabilities)
 import           System.Directory (removeFile)
 
@@ -147,15 +148,23 @@ tMinMax [] = fail "tMinMax: empty list of log files"
 tMinMax [log] = do
   ls2 <- BSL.lines <$> BSL.readFile log
   let
-    loMin, loMax :: LogObject
+    loMin :: LogObject
+    loMax :: UTCTime
     loMin = head $ mapMaybe Aeson.decode ls2
-    loMax = fromJust (Aeson.decode $ last ls2)
-  pure (loAt loMin, loAt loMax)
+    loMax = fromJust $ (loAt <$> Aeson.decode (last ls2)) <|> parseTimeStampFromPartial (last ls2)
+  pure (loAt loMin, loMax)
 tMinMax logs = do
   (tMin, _   ) <- tMinMax [head logs]
   (_   , tMax) <- tMinMax [last logs]
   pure (tMin, tMax)
 
+-- extracts timestamp from the last log line even if incomplete (due to node shutdown) - sample data:
+-- {"at":"2026-03-25T16:19:47.482228445Z","ns":"...",...
+parseTimeStampFromPartial :: BSL.ByteString -> Maybe UTCTime
+parseTimeStampFromPartial partial =
+  formatParseM iso8601Format $ BSL.unpack ts
+  where ts = BSL.takeWhile (/= '"') . BSL.drop 7 $ partial
+
 
 -- selects the entire LogObject stream, containing all objects relevant for standard analysis
 selectAll :: SQL

bench/tx-generator/src/Cardano/Benchmarking/Tracer.hs

@@ -182,19 +182,15 @@ configSilent :: ConfigOption
 configSilent = ConfSeverity (SeverityF Nothing)
 
 initialTraceConfig :: TraceConfig
-initialTraceConfig = TraceConfig {
-      tcOptions = Map.fromList
+initialTraceConfig = emptyTraceConfig 
+    { tcOptions = Map.fromList
           [ ([], [configSilent])
           , setMaxDetail TracerNameBench
           , ([TracerNameSubmitN2N], [configSilent])
           , setMaxDetail TracerNameConnect
           , setMaxDetail TracerNameSubmit
           ]
     , tcForwarder = Just defaultForwarder {tofQueueSize = 4096}
-    , tcNodeName = Nothing
-    , tcResourceFrequency = Nothing
-    , tcMetricsPrefix = Nothing
-    , tcLedgerMetricsFrequency = Nothing
     }
   where
     setMaxDetail :: Text -> ([Text], [ConfigOption])

cardano-node/src/Cardano/Node/Tracing/DefaultTraceConfig.hs

@@ -32,8 +32,6 @@ defaultCardanoConfig = emptyTraceConfig {
           [ ConfSeverity (SeverityF (Just Warning))])
      ,(["Net", "ConnectionManager", "Remote"],
           [ ConfSeverity (SeverityF (Just Info))])
-     ,(["Net", "Subscription", "DNS"],
-          [ ConfSeverity (SeverityF (Just Info))])
      ,(["Startup", "DiffusionInit"],
           [ ConfSeverity (SeverityF (Just Info))])
      ,(["Net", "ErrorPolicy"],
@@ -44,10 +42,6 @@ defaultCardanoConfig = emptyTraceConfig {
           [ ConfSeverity (SeverityF (Just Info))])
      ,(["Net", "InboundGovernor", "Remote"],
           [ ConfSeverity (SeverityF (Just Info))])
-     ,(["Net", "Subscription", "IP"],
-          [ ConfSeverity (SeverityF (Just Info))])
-     ,(["Net", "ErrorPolicy", "Local"],
-          [ ConfSeverity (SeverityF (Just Info))])
      ,(["Mempool"],
           [ ConfSeverity (SeverityF (Just Info))])
      ,(["Net", "Mux", "Remote"],

cardano-node/src/Cardano/Node/Tracing/Tracers/BlockReplayProgress.hs

@@ -19,6 +19,8 @@ import           Ouroboros.Network.Point (withOrigin)
 import           Control.Concurrent.MVar
 import           Data.Aeson (Value (String), (.=))
 import           Data.Text (pack)
+import           Numeric (showFFloat)
+
 
 newtype ReplayBlockState = ReplayBlockState
   { -- | Last slot for which a `ReplayBlockStats` message has been issued.
@@ -63,7 +65,7 @@ instance LogFormatting ReplayBlockStats where
       <> " out of "
       <> textShow (unSlotNo rpsGoalSlot)
       <> ". Progress: "
-      <> textShow (progressForHuman stats)
+      <> pack (showFFloat (Just 2) (progressForHuman stats) mempty)
       <> "%"
 
   asMetrics stats =

cardano-tracer/src/Cardano/Tracer/MetaTrace.hs

@@ -292,14 +292,8 @@ configTracerTracer defSeverity tr = do
   configureTracers configReflection initialTraceConfig [tr]
  where
    initialTraceConfig :: TraceConfig
-   initialTraceConfig =
-     TraceConfig
-     { tcForwarder         = Nothing
-     , tcNodeName          = Nothing
-     , tcResourceFrequency = Nothing
-     , tcLedgerMetricsFrequency = Nothing
-     , tcMetricsPrefix     = Nothing
-     , tcOptions = Map.fromList
+   initialTraceConfig = emptyTraceConfig
+     { tcOptions = Map.fromList
                    [ ([],         [ConfSeverity defSeverity])
                    , (["Tracer"], [ConfDetail DMaximum])
                    ]

nix/workbench/service/tracing.nix

@@ -9,12 +9,18 @@ with lib;
 let
   trace-dispatcher =
     recursiveUpdate
-    (removeLegacyTracingOptions cfg)
+    (removeAttrs (removeLegacyTracingOptions cfg) ["TraceOptionForwarder"])
   {
     UseTraceDispatcher   = true;
     TraceOptionResourceFrequency = 1000;
     TraceOptionNodeName = nodeSpec.name;
 
+  ## Set this value to cover our log line rate (~70Hz) on typical bench machines (not the apex hardware).
+  ## It should avoid seeing "TraceObject queue overflowed. Dropped xxx messages ..." on a bench node's stderr.
+    TraceOptionForwarder = {
+      queueSize = 192;
+    };
+
   ## Please see the generated tracing configuration reference at:
   ##
   ## https://github.com/intersectmbo/cardano-node/blob/master/doc/new-tracing/tracers_doc_generated.md#trace-messages
@@ -38,7 +44,7 @@ let
       "BlockFetch.Server".severity = "Debug";
       "BlockchainTime".severity = "Notice";
       "ChainDB".severity = "Debug";
-      "ChainDB.LedgerEvent.Flavor.V1.OnDisk.BackingStoreEvent".severity = "Silence";
+      "ChainDB.LedgerEvent.Flavor.V1.".severity = "Silence";
       "ChainDB.LedgerEvent.Flavor.V2".severity = "Silence";
       "ChainDB.ReplayBlock.LedgerReplay".severity = "Notice";
       "ChainSync.Client".severity = "Debug";
@@ -57,11 +63,11 @@ let
       "Net.ConnectionManager.Local".severity = "Debug";
       "Net.ConnectionManager.Remote".severity = "Debug";
       "Net.DNSResolver".severity = "Notice";
-      "Net.ErrorPolicy.Local".severity = "Debug";
-      "Net.ErrorPolicy.Remote".severity = "Debug";
+      "Net.ErrorPolicy".severity = "Debug";
       "Net.Handshake.Local".severity = "Debug";
       "Net.Handshake.Remote".severity = "Debug";
       "Net.InboundGovernor.Local".severity = "Debug";
+      "Net.InboundGovernor.Local.RemoteState".severity = "Info";
       "Net.InboundGovernor.Remote".severity = "Debug";
       "Net.InboundGovernor.Transition".severity = "Debug";
       "Net.Mux.Local".severity = "Notice";
@@ -73,13 +79,10 @@ let
       "Net.PeerSelection.Responder".severity = "Notice";
       "Net.PeerSelection.Selection".severity = "Debug";
       "Net.Peers.Ledger".severity = "Debug";
-      "Net.Peers.List".severity = "Notice";
       "Net.Peers.LocalRoot".severity = "Debug";
       "Net.Peers.PublicRoot".severity = "Debug";
       "Net.Server.Local".severity = "Debug";
       "Net.Server.Remote".severity = "Debug";
-      "Net.Subscription.DNS".severity = "Debug";
-      "Net.Subscription.IP".severity = "Debug";
       "NodeState".severity = "Notice";
       "LedgerMetrics".severity = "Info";
       "Resources".severity = "Debug";

shell.nix

@@ -181,28 +181,37 @@ let
 
   haveGlibcLocales = pkgs.glibcLocales != null && stdenv.hostPlatform.libc == "glibc";
 
+  workbench = import ./nix/workbench/default.nix
+                { inherit pkgs; haskellProject = project; }
+  ;
+
   workbench-shell =
     with customConfig.localCluster;
       import ./nix/workbench/shell.nix
         { inherit pkgs lib haskellLib project;
           inherit setLocale haveGlibcLocales;
           inherit workbenchDevMode;
           inherit withHoogle;
-          workbench-runner = pkgs.workbench-runner
+          workbench-runner = workbench.runner
             { inherit profiling;
-              inherit profileName backendName useCabalRun;
+              inherit profileName backendName;
+              inherit useCabalRun;
+              inherit workbenchStartArgs cardano-node-rev;
+              inherit (customConfig.localCluster) stateDir basePort batchName;
             };
         };
 
   devops =
     let profileName = "devops-bage";
-        workbench-runner = pkgs.workbench-runner
+        workbench-runner = workbench.runner
           { inherit profiling;
             inherit profileName;
             backendName = "supervisor";
             useCabalRun = false;
+            inherit workbenchStartArgs cardano-node-rev;
+            inherit (customConfig.localCluster) stateDir basePort batchName;
           };
-        devopsShell = with customConfig.localCluster;
+        devopsShell = 
           import ./nix/workbench/shell.nix
             { inherit pkgs lib haskellLib project;
               inherit setLocale haveGlibcLocales;