fix: apply core sub-config overrides during startup parsing

fixcore/fixcore/__main__.py

@@ -33,7 +33,6 @@
     inside_docker,
     inside_kubernetes,
     helm_installation,
-    FixCoreConfigId,
     parse_config,
     CoreConfig,
 )
@@ -123,7 +122,7 @@ def run_process(args: Namespace) -> None:
         if args.multi_tenant_setup:
             deps = Dependencies(system_info=system_info())
             deps.add(ServiceNames.temp_dir, temp)
-            config = deps.add(ServiceNames.config, parse_config(args, {}, lambda: None))
+            config = deps.add(ServiceNames.config, parse_config(args, {}, lambda _: None))
             # jwt_signing_keys are not required for multi-tenant setup.
             deps.add(ServiceNames.jwt_signing_key_holder, EphemeralJwtSigningKey())
             cert_handler_no_ca = deps.add(ServiceNames.cert_handler, CertificateHandlerNoCA.lookup(config, temp))
@@ -146,7 +145,7 @@ def run_process(args: Namespace) -> None:
                 deps.add(ServiceNames.system_data, system_data)
                 # only to be used for CoreConfig creation
                 core_config_override_service = asyncio.run(override_config_for_startup(args.config_override_path))
-                config = config_from_db(args, sdb, lambda: core_config_override_service.get_override(FixCoreConfigId))
+                config = config_from_db(args, sdb, core_config_override_service.get_override)
                 cert_handler = deps.add(ServiceNames.cert_handler, CertificateHandlerWithCA.lookup(config, sdb, temp))
                 verify = False if args.graphdb_no_ssl_verify else str(cert_handler.ca_bundle)
                 deps.add(ServiceNames.config, evolve(config, run=RunConfig(temp, verify)))

fixcore/fixcore/config/config_handler_service.py

@@ -207,7 +207,8 @@ def overridden_parts(existing: JsonElement, update: JsonElement) -> JsonElement:
 
                     def mkstr(val: Any) -> str:
                         if isinstance(val, list):
-                            return f'[{", ".join(val)}]'
+                            items = cast(List[object], val)
+                            return f'[{", ".join(str(v) for v in items)}]'
                         return str(val)
 
                     return mkstr(update)

fixcore/fixcore/core_config.py

@@ -697,7 +697,7 @@ def config_model() -> List[Json]:
 def parse_config(
     args: Namespace,
     core_config: Json,
-    get_core_overrides: Callable[[], Optional[Json]],
+    get_overrides: Callable[[ConfigId], Optional[Json]],
     command_templates: Optional[Json] = None,
     snapshot_schedule: Optional[Json] = None,
 ) -> CoreConfig:
@@ -730,7 +730,7 @@ def parse_config(
             adjusted = set_value_in_path(value, path, adjusted)
 
     # here we only care about the fixcore overrides
-    core_config_overrides = (get_core_overrides() or {}).get(FixCoreRoot)
+    core_config_overrides = (get_overrides(FixCoreConfigId) or {}).get(FixCoreRoot)
     # merge the file overrides into the adjusted config
     if core_config_overrides:
         adjusted = merge_json_elements(adjusted, core_config_overrides)
@@ -756,20 +756,32 @@ def parse_config(
         ed = EditableConfig()
 
     commands_config = CustomCommandsConfig()
-    if command_templates:
-        try:
-            migrated_commands = migrate_command_config(command_templates)
-            cmd_cfg_to_parse = migrated_commands or command_templates
-            commands_config = from_js(cmd_cfg_to_parse.get(FixCoreCommandsRoot), CustomCommandsConfig)
-        except Exception as e:
-            log.error(f"Can not parse command templates. Fall back to defaults. Reason: {e}", exc_info=e)
+    try:
+        command_config_json = command_templates or CustomCommandsConfig().json()
+        migrated_commands = migrate_command_config(command_config_json)
+        cmd_cfg_to_parse = migrated_commands or command_config_json
+        command_overrides = (get_overrides(FixCoreCommandsConfigId) or {}).get(FixCoreCommandsRoot)
+        if command_overrides:
+            cmd_cfg_to_parse = cast(
+                Json,
+                merge_json_elements(cmd_cfg_to_parse, {FixCoreCommandsRoot: command_overrides}),
+            )
+        commands_config = from_js(cmd_cfg_to_parse.get(FixCoreCommandsRoot), CustomCommandsConfig)
+    except Exception as e:
+        log.error(f"Can not parse command templates. Fall back to defaults. Reason: {e}", exc_info=e)
 
     snapshots_config = SnapshotsScheduleConfig()
-    if snapshot_schedule:
-        try:
-            snapshots_config = from_js(snapshot_schedule.get(FixCoreSnapshotsRoot), SnapshotsScheduleConfig)
-        except Exception as e:
-            log.error(f"Can not parse snapshot schedule. Fall back to defaults. Reason: {e}", exc_info=e)
+    try:
+        snapshot_config_json = snapshot_schedule or SnapshotsScheduleConfig().json()
+        snapshot_overrides = (get_overrides(FixCoreSnapshotsConfigId) or {}).get(FixCoreSnapshotsRoot)
+        if snapshot_overrides:
+            snapshot_config_json = cast(
+                Json,
+                merge_json_elements(snapshot_config_json, {FixCoreSnapshotsRoot: snapshot_overrides}),
+            )
+        snapshots_config = from_js(snapshot_config_json.get(FixCoreSnapshotsRoot), SnapshotsScheduleConfig)
+    except Exception as e:
+        log.error(f"Can not parse snapshot schedule. Fall back to defaults. Reason: {e}", exc_info=e)
 
     return CoreConfig(
         api=ed.api,
@@ -818,7 +830,7 @@ def migrate_command_config(cmd_config: Json) -> Optional[Json]:
 def config_from_db(
     args: Namespace,
     db: StandardDatabase,
-    get_core_overrides: Callable[[], Optional[Json]],
+    get_overrides: Callable[[ConfigId], Optional[Json]],
     collection_name: str = "configs",
 ) -> CoreConfig:
     if configs := db.collection(collection_name) if db.has_collection(collection_name) else None:
@@ -830,5 +842,5 @@ def config_from_db(
                 snapshots_config_entity = cast(Optional[Json], configs.get(FixCoreSnapshotsConfigId))
                 snapshots_config = snapshots_config_entity.get("config") if snapshots_config_entity else None
 
-                return parse_config(args, config, get_core_overrides, command_config, snapshots_config)
-    return parse_config(args, {}, get_core_overrides)
+                return parse_config(args, config, get_overrides, command_config, snapshots_config)
+    return parse_config(args, {}, get_overrides)

fixcore/fixcore/system_start.py

@@ -243,7 +243,7 @@ def key_value(kv: str) -> Tuple[str, JsonElement]:
 
 
 def empty_config(args: Optional[List[str]] = None) -> CoreConfig:
-    return parse_config(parse_args(args or []), {}, lambda: None)
+    return parse_config(parse_args(args or []), {}, lambda _: None)
 
 
 # Note: this method should be called from every started process as early as possible

fixcore/tests/fixcore/config/config_handler_service_test.py

@@ -8,6 +8,7 @@
 from fixcore.analytics import InMemoryEventSender
 from fixcore.config import ConfigHandler, ConfigEntity, ConfigValidation, ConfigOverride
 from fixcore.config.config_handler_service import ConfigHandlerService
+from fixcore.core_config import CustomCommandsConfig, FixCoreCommandsConfigId
 from fixcore.ids import ConfigId
 from fixcore.message_bus import CoreMessage, Event, Message
 from tests.fixcore.message_bus_test import wait_for_message
@@ -300,6 +301,37 @@ async def test_config_yaml(config_handler: ConfigHandler, config_model: List[Kin
     assert expect_override_comment in config_with_override_yaml
 
 
+@pytest.mark.asyncio
+async def test_config_yaml_handles_non_string_list_overrides_for_core_commands(
+    config_handler: ConfigHandler,
+) -> None:
+    override = {
+        "custom_commands": {
+            "commands": [
+                {
+                    "name": "search-foo",
+                    "info": "Search Foo resources.",
+                    "template": "search is(foo)",
+                }
+            ]
+        }
+    }
+
+    await config_handler.put_config(
+        ConfigEntity(FixCoreCommandsConfigId, CustomCommandsConfig().json()), validate=False
+    )
+    cast(ConfigHandlerService, config_handler).override_service = cast(
+        ConfigOverride,
+        SimpleNamespace(get_override=lambda cfg_id: override if cfg_id == FixCoreCommandsConfigId else None),
+    )
+
+    config_yaml = await config_handler.config_yaml(FixCoreCommandsConfigId)
+
+    assert config_yaml is not None
+    assert "custom_commands:" in config_yaml
+    assert "commands:" in config_yaml
+
+
 @pytest.mark.asyncio
 async def test_config_change_emits_event(config_handler: ConfigHandler, all_events: List[Message]) -> None:
     # Put a config

fixcore/tests/fixcore/core_config_test.py

@@ -16,9 +16,13 @@
     migrate_core_config,
     migrate_command_config,
     CustomCommandsConfig,
+    SnapshotsScheduleConfig,
     alias_templates,
+    FixCoreCommandsConfigId,
     FixCoreCommandsRoot,
     FixCoreConfigId,
+    FixCoreSnapshotsConfigId,
+    FixCoreSnapshotsRoot,
     current_git_hash,
 )
 from fixcore.system_start import parse_args
@@ -40,7 +44,7 @@ def test_parse_broken(config_json: Json) -> None:
     del cfg["fixcore"]["api"]["https_port"]
 
     # parse this configuration
-    parsed = parse_config(parse_args(["--analytics-opt-out"]), cfg, lambda: None)
+    parsed = parse_config(parse_args(["--analytics-opt-out"]), cfg, lambda _: None)
     parsed_json = to_js(parsed.editable, strip_attr="kind")
 
     # web_hosts and https_port were not available and are reverted to the default values
@@ -57,13 +61,13 @@ def test_parse_broken(config_json: Json) -> None:
 
 
 def test_read_config(config_json: Json) -> None:
-    parsed = parse_config(parse_args(["--analytics-opt-out"]), config_json, lambda: None)
+    parsed = parse_config(parse_args(["--analytics-opt-out"]), config_json, lambda _: None)
     assert parsed.json() == config_json
 
 
 def test_override_via_cmd_line(default_config: CoreConfig) -> None:
     config = {"runtime": {"debug": False}}
-    parsed = parse_config(parse_args(["--debug"]), config, lambda: None)
+    parsed = parse_config(parse_args(["--debug"]), config, lambda _: None)
     assert parsed.runtime.debug == True
 
 
@@ -101,12 +105,57 @@ def test_config_override(config_json: Json) -> None:
     parsed = parse_config(
         parse_args(["--analytics-opt-out"]),
         cfg,
-        lambda: overrides.get(FixCoreConfigId),
+        lambda config_id: overrides.get(config_id),
     )
     assert parsed.api.web_hosts == ["11.12.13.14"]
     assert parsed.api.https_port == 1337
 
 
+def test_command_config_override() -> None:
+    overrides = {
+        FixCoreCommandsConfigId: {
+            FixCoreCommandsRoot: {
+                "commands": [
+                    {
+                        "name": "compliance_orphan_disk_match",
+                        "info": "Read-only selector for disks without an attached consumer signal.",
+                        "template": "search is(volume) with (empty, <-- is(instance))",
+                    }
+                ]
+            }
+        }
+    }
+
+    parsed = parse_config(
+        parse_args(["--analytics-opt-out"]),
+        {},
+        lambda config_id: overrides.get(config_id),
+    )
+
+    assert any(cmd.name == "compliance_orphan_disk_match" for cmd in parsed.custom_commands.commands)
+
+
+def test_snapshot_config_override() -> None:
+    overrides = {
+        FixCoreSnapshotsConfigId: {
+            FixCoreSnapshotsRoot: {
+                "snapshots": {
+                    "weekly": {"schedule": "0 12 * * 1", "retain": 9},
+                }
+            }
+        }
+    }
+
+    parsed = parse_config(
+        parse_args(["--analytics-opt-out"]),
+        {},
+        lambda config_id: overrides.get(config_id),
+    )
+
+    assert parsed.snapshots.snapshots["weekly"].schedule == "0 12 * * 1"
+    assert parsed.snapshots.snapshots["weekly"].retain == 9
+
+
 def test_model() -> None:
     model = config_model()
     assert {m["fqn"] for m in model} == {