gRPC: Reload configuration on SIGHUP

cardano-node/src/Cardano/Node/Run.hs

@@ -43,8 +43,8 @@ import           Cardano.Node.Configuration.POM (NodeConfiguration (..),
                    defaultPartialNodeConfiguration, makeNodeConfiguration,
                    parseNodeConfigurationFP, getForkPolicy)
 import           Cardano.Node.Configuration.Socket (LocalSocketOrSocketInfo,
-                   SocketOrSocketInfo, SocketOrSocketInfo' (..),
-                   gatherConfiguredSockets, getSocketOrSocketInfoAddr)
+                   SocketOrSocketInfo, SocketOrSocketInfo' (..), gatherConfiguredSockets,
+                   getSocketOrSocketInfoAddr)
 import           Cardano.Node.Configuration.TopologyP2P
 import qualified Cardano.Node.Configuration.TopologyP2P as TopologyP2P
 import           Cardano.Node.Handlers.Shutdown
@@ -143,7 +143,9 @@ import           Control.Monad.Trans.Except.Extra (left, hushM)
 import           Control.Monad.Trans.Maybe (MaybeT(runMaybeT, MaybeT), hoistMaybe)
 import           "contra-tracer" Control.Tracer
 import           Data.Bits
+import           Data.Bifunctor (first)
 import           Data.Either (partitionEithers)
+import           Data.Functor.Identity (Identity (..))
 import           Data.IP (toSockAddr)
 import           Data.Map.Strict (Map)
 import qualified Data.Map.Strict as Map
@@ -481,6 +483,7 @@ handleSimpleNode blockType runP tracers nc networkMagic onKernel = do
                                             (readTVar ledgerPeerSnapshotPathVar)
                                             (readTVar useLedgerVar)
                                             (const . pure $ ())
+    rpcConfigVar <- newTVarIO (ncRpcConfig nc)
 
     let nodeArgs = RunNodeArgs
           { rnGenesisConfig  = ncGenesisConfig nc
@@ -513,8 +516,8 @@ handleSimpleNode blockType runP tracers nc networkMagic onKernel = do
           , rnFeatureFlags = mempty -- TODO(10.7) forward this to CLI options?
           }
 #ifdef UNIX
-    -- initial `SIGHUP` handler, which only rereads the topology file but
-    -- doesn't update block forging.  The latter is only possible once
+    -- initial `SIGHUP` handler, which rereads the topology file and the RPC config from the main configuration file
+    -- but doesn't update block forging. The latter is only possible once
     -- consensus initialised (e.g. reapplied all blocks).
     _ <- Signals.installHandler
           Signals.sigHUP
@@ -531,6 +534,7 @@ handleSimpleNode blockType runP tracers nc networkMagic onKernel = do
               (readTVar ledgerPeerSnapshotPathVar)
               (readTVar useLedgerVar)
               (writeTVar ledgerPeerSnapshotVar)
+            updateRpcConfiguration (startupTracer tracers) (ncConfigFile nc) rpcConfigVar
             traceWith (startupTracer tracers) (BlockForgingUpdate NotEffective)
           )
           Nothing
@@ -569,14 +573,33 @@ handleSimpleNode blockType runP tracers nc networkMagic onKernel = do
               (readTVar ledgerPeerSnapshotVar)
               nc
       in
-      withAsync (runRpcServer (rpcTracer tracers) (ncRpcConfig nc, networkMagic))  $ \_ ->
+      let waitForRpcConfigChange oldConfig = atomically $ readTVar rpcConfigVar >>= \new -> check (new /= oldConfig)
+          disableRpcServer = atomically . modifyTVar rpcConfigVar $ \config -> config{isEnabled = Identity False}
+          rpcServerLoop = do
+            config@RpcConfig{isEnabled = Identity enabled} <- readTVarIO rpcConfigVar
+            if enabled
+              then
+                race_
+                  -- Run server. If @runRpcServer@ exits, means that we got (and handled already) a fatal error.
+                  -- To prevent looping infinitely here, we disable RPC server. This is recoverable if the user
+                  -- sends SIGHUP to reload the config and reenable the server.
+                  (do 
+                    runRpcServer (rpcTracer tracers) (config, networkMagic)
+                    traceWith (startupTracer tracers) RpcForceDisabled
+                    disableRpcServer)
+                  (waitForRpcConfigChange config)
+              else waitForRpcConfigChange config
+            rpcServerLoop
+      in
+      withAsync rpcServerLoop $ \_ ->
         Node.run
           nodeArgs {
               rnNodeKernelHook = \registry nodeKernel -> do
                 -- reinstall `SIGHUP` handler
                 installSigHUPHandler (startupTracer tracers) (Consensus.kesAgentTracer $ consensusTracers tracers)
                                      blockType nc networkMagic nodeKernel localRootsVar publicRootsVar useLedgerVar
                                      useBootstrapVar ledgerPeerSnapshotPathVar ledgerPeerSnapshotVar
+                                     rpcConfigVar
                 rnNodeKernelHook nodeArgs registry nodeKernel
           }
           StdRunNodeArgs
@@ -664,8 +687,7 @@ handleSimpleNode blockType runP tracers nc networkMagic onKernel = do
 -- SIGHUP Handlers
 --------------------------------------------------------------------------------
 
--- | The P2P SIGHUP handler can update block forging & reconfigure network topology.
---
+-- | The P2P SIGHUP handler can update block forging, reconfigure network topology and restart gRPC.
 installSigHUPHandler :: Tracer IO (StartupTrace blk)
                      -> Tracer IO KESAgentClientTrace
                      -> Api.BlockType blk
@@ -678,12 +700,14 @@ installSigHUPHandler :: Tracer IO (StartupTrace blk)
                      -> StrictTVar IO UseBootstrapPeers
                      -> StrictTVar IO (Maybe PeerSnapshotFile)
                      -> StrictTVar IO (Maybe (LedgerPeerSnapshot BigLedgerPeers))
+                     -> StrictTVar IO RpcConfig
                      -> IO ()
 #ifndef UNIX
-installSigHUPHandler _ _ _ _ _ _ _ _ _ _ _ _ = return ()
+installSigHUPHandler _ _ _ _ _ _ _ _ _ _ _ _ _ = return ()
 #else
 installSigHUPHandler startupTracer kesAgentTracer blockType nc networkMagic nodeKernel localRootsVar
-                     publicRootsVar useLedgerVar useBootstrapPeersVar ledgerPeerSnapshotPathVar ledgerPeerSnapshotVar =
+                     publicRootsVar useLedgerVar useBootstrapPeersVar ledgerPeerSnapshotPathVar ledgerPeerSnapshotVar
+                     rpcConfigVar =
   void $ Signals.installHandler
     Signals.sigHUP
     (Signals.Catch $ do
@@ -698,6 +722,7 @@ installSigHUPHandler startupTracer kesAgentTracer blockType nc networkMagic node
                (readTVar ledgerPeerSnapshotPathVar)
                (readTVar useLedgerVar)
                (writeTVar ledgerPeerSnapshotVar)
+      updateRpcConfiguration startupTracer (ncConfigFile nc) rpcConfigVar
     )
     Nothing
 #endif
@@ -842,6 +867,33 @@ updateLedgerPeerSnapshot startupTracer NodeConfiguration { ncConsensusMode } net
 
   mLedgerPeerSnapshot <$ atomically (writeVar mLedgerPeerSnapshot)
 
+#ifdef UNIX
+-- | Reload RPC configuration from the configuration file
+updateRpcConfiguration :: Tracer IO (StartupTrace blk) -- ^ tracer tracing the configuration reload
+                       -> ConfigYamlFilePath -- ^ node configuration file, to reload configuration from
+                       -> StrictTVar IO RpcConfig -- ^ TVar storing RPC configuration
+                       -> IO ()
+updateRpcConfiguration tracer configFilePath rpcConfigVar = do
+  result <- fmap (join . first Exception.displayException)
+              . try @Exception.SomeException
+              . fmap makeNodeConfiguration
+              . parseNodeConfigurationFP
+              $ Just configFilePath
+  case result of
+    Left err ->
+      -- reload failure, we don't do anything this time
+      traceWith tracer (RpcConfigUpdateError $ pack err)
+    Right NodeConfiguration{ncRpcConfig=newConfig} ->
+      join . atomically $ do
+        oldConfig <- readTVar rpcConfigVar
+        if oldConfig /= newConfig
+          then do
+            writeTVar rpcConfigVar newConfig
+            pure . traceWith tracer . RpcConfigUpdate . pack $ show newConfig
+          else
+            pure $ pure ()
+#endif
+
 --------------------------------------------------------------------------------
 -- Helper functions
 --------------------------------------------------------------------------------

cardano-node/src/Cardano/Node/Startup.hs

@@ -141,6 +141,13 @@ data StartupTrace blk =
   | BIByron BasicInfoByron
   | BINetwork BasicInfoNetwork
   | LedgerPeerSnapshotLoaded SlotNo
+  -- | Log that RPC configuration has been updated
+  | RpcConfigUpdate Text
+  -- | Log RPC configuration update error
+  | RpcConfigUpdateError Text
+  -- | Log RPC is forcefully disabled after a RPC server crash.
+  | RpcForceDisabled
+
   | MovedTopLevelOption String
 
 data LedgerPeerSnapshotError = LedgerPeerSnapshotTooOld SlotNo SlotNo PeerSnapshotFile

cardano-node/src/Cardano/Node/Tracing/Tracers/Startup.hs

@@ -282,6 +282,16 @@ instance ( Show (BlockNodeToNodeVersion blk)
                , "commit" .= String biCommit
                , "nodeStartTime" .= biNodeStartTime
                ]
+  forMachine _dtal (RpcConfigUpdate config) =
+      mconcat [ "kind" .= String "RpcConfigUpdate"
+               , "message" .= String "RPC configuration update"
+               , "configuration" .= String config ]
+  forMachine _dtal (RpcConfigUpdateError err) =
+      mconcat [ "kind" .= String "RpcConfigUpdateError"
+               , "error" .= String err ]
+  forMachine _dtal RpcForceDisabled =
+      mconcat [ "kind" .= String "RpcForceDisabled"
+               , "error" .= String (ppStartupInfoTrace RpcForceDisabled)]
   forMachine _dtal (MovedTopLevelOption opt) =
       mconcat [ "kind" .= String "MovedTopLevelOption"
               , "option" .= opt
@@ -347,6 +357,12 @@ instance MetaTrace  (StartupTrace blk) where
     Namespace [] ["Byron"]
   namespaceFor BINetwork {}  =
     Namespace [] ["Network"]
+  namespaceFor RpcConfigUpdate {} =
+    Namespace [] ["RpcConfigUpdate"]
+  namespaceFor RpcConfigUpdateError {} =
+    Namespace [] ["RpcConfigUpdateError"]
+  namespaceFor RpcForceDisabled =
+    Namespace [] ["RpcForceDisabled"]
   namespaceFor MovedTopLevelOption {} =
     Namespace [] ["MovedTopLevelOption"]
 
@@ -359,6 +375,9 @@ instance MetaTrace  (StartupTrace blk) where
   severityFor (Namespace _ ["NonP2PWarning"]) _ = Just Warning
   severityFor (Namespace _ ["WarningDevelopmentNodeToNodeVersions"]) _ = Just Warning
   severityFor (Namespace _ ["WarningDevelopmentNodeToClientVersions"]) _ = Just Warning
+  severityFor (Namespace _ ["RpcConfigUpdate"]) _ = Just Notice
+  severityFor (Namespace _ ["RpcConfigUpdateError"]) _ = Just Error
+  severityFor (Namespace _ ["RpcForceDisabled"]) _ = Just Error
   severityFor (Namespace _ ["BlockForgingUpdateError"]) _ = Just Error
   severityFor (Namespace _ ["BlockForgingBlockTypeMismatch"]) _ = Just Error
   severityFor (Namespace _ ["MovedTopLevelOption"]) _ = Just Warning
@@ -385,6 +404,12 @@ instance MetaTrace  (StartupTrace blk) where
     ""
   documentFor (Namespace [] ["BlockForgingBlockTypeMismatch"]) = Just
     ""
+  documentFor (Namespace [] ["RpcConfigUpdate"]) = Just
+    ""
+  documentFor (Namespace [] ["RpcConfigUpdateError"]) = Just
+    ""
+  documentFor (Namespace [] ["RpcForceDisabled"]) = Just
+    ""
   documentFor (Namespace [] ["NetworkConfigUpdate"]) = Just
     ""
   documentFor (Namespace [] ["NetworkConfigUpdateUnsupported"]) = Just
@@ -454,6 +479,9 @@ instance MetaTrace  (StartupTrace blk) where
     , Namespace [] ["DBValidation"]
     , Namespace [] ["BlockForgingUpdate"]
     , Namespace [] ["BlockForgingBlockTypeMismatch"]
+    , Namespace [] ["RpcConfigUpdate"]
+    , Namespace [] ["RpcConfigUpdateError"]
+    , Namespace [] ["RpcForceDisabled"]
     , Namespace [] ["NetworkConfigUpdate"]
     , Namespace [] ["NetworkConfigUpdateUnsupported"]
     , Namespace [] ["NetworkConfigUpdateError"]
@@ -576,6 +604,10 @@ ppStartupInfoTrace (NetworkConfig localRoots publicRoots useLedgerPeers peerSnap
 ppStartupInfoTrace (LedgerPeerSnapshotLoaded slotNo) =
     "Topology: Peer snapshot containing ledger peers recorded at " <> showT slotNo <> " loaded."
 
+ppStartupInfoTrace (RpcConfigUpdate config) = "Performing RPC configuration update: " <> config
+ppStartupInfoTrace (RpcConfigUpdateError err) = "Error while updating RPC configuration: " <> err
+ppStartupInfoTrace RpcForceDisabled = "RPC endpoint has crashed and because of that it got disabled. Enable gRPC endpoint and send SIGHUP to the node to reenable."
+
 ppStartupInfoTrace NonP2PWarning = nonP2PWarningMessage
 
 ppStartupInfoTrace (WarningDevelopmentNodeToNodeVersions ntnVersions) =