diff --git a/fixcore/fixcore/__main__.py b/fixcore/fixcore/__main__.py index ede3e62b59..c780123a23 100644 --- a/fixcore/fixcore/__main__.py +++ b/fixcore/fixcore/__main__.py @@ -33,7 +33,6 @@ inside_docker, inside_kubernetes, helm_installation, - FixCoreConfigId, parse_config, CoreConfig, ) @@ -123,7 +122,7 @@ def run_process(args: Namespace) -> None: if args.multi_tenant_setup: deps = Dependencies(system_info=system_info()) deps.add(ServiceNames.temp_dir, temp) - config = deps.add(ServiceNames.config, parse_config(args, {}, lambda: None)) + config = deps.add(ServiceNames.config, parse_config(args, {}, lambda _: None)) # jwt_signing_keys are not required for multi-tenant setup. deps.add(ServiceNames.jwt_signing_key_holder, EphemeralJwtSigningKey()) cert_handler_no_ca = deps.add(ServiceNames.cert_handler, CertificateHandlerNoCA.lookup(config, temp)) @@ -146,7 +145,7 @@ def run_process(args: Namespace) -> None: deps.add(ServiceNames.system_data, system_data) # only to be used for CoreConfig creation core_config_override_service = asyncio.run(override_config_for_startup(args.config_override_path)) - config = config_from_db(args, sdb, lambda: core_config_override_service.get_override(FixCoreConfigId)) + config = config_from_db(args, sdb, core_config_override_service.get_override) cert_handler = deps.add(ServiceNames.cert_handler, CertificateHandlerWithCA.lookup(config, sdb, temp)) verify = False if args.graphdb_no_ssl_verify else str(cert_handler.ca_bundle) deps.add(ServiceNames.config, evolve(config, run=RunConfig(temp, verify))) diff --git a/fixcore/fixcore/config/config_handler_service.py b/fixcore/fixcore/config/config_handler_service.py index cf68d6a81d..9e54543ba2 100644 --- a/fixcore/fixcore/config/config_handler_service.py +++ b/fixcore/fixcore/config/config_handler_service.py @@ -207,7 +207,8 @@ def overridden_parts(existing: JsonElement, update: JsonElement) -> JsonElement: def mkstr(val: Any) -> str: if isinstance(val, list): - return f'[{", ".join(val)}]' + items = cast(List[object], val) + return f'[{", ".join(str(v) for v in items)}]' return str(val) return mkstr(update) diff --git a/fixcore/fixcore/core_config.py b/fixcore/fixcore/core_config.py index 9da6e746b8..be18310667 100644 --- a/fixcore/fixcore/core_config.py +++ b/fixcore/fixcore/core_config.py @@ -697,7 +697,7 @@ def config_model() -> List[Json]: def parse_config( args: Namespace, core_config: Json, - get_core_overrides: Callable[[], Optional[Json]], + get_overrides: Callable[[ConfigId], Optional[Json]], command_templates: Optional[Json] = None, snapshot_schedule: Optional[Json] = None, ) -> CoreConfig: @@ -730,7 +730,7 @@ def parse_config( adjusted = set_value_in_path(value, path, adjusted) # here we only care about the fixcore overrides - core_config_overrides = (get_core_overrides() or {}).get(FixCoreRoot) + core_config_overrides = (get_overrides(FixCoreConfigId) or {}).get(FixCoreRoot) # merge the file overrides into the adjusted config if core_config_overrides: adjusted = merge_json_elements(adjusted, core_config_overrides) @@ -756,20 +756,32 @@ def parse_config( ed = EditableConfig() commands_config = CustomCommandsConfig() - if command_templates: - try: - migrated_commands = migrate_command_config(command_templates) - cmd_cfg_to_parse = migrated_commands or command_templates - commands_config = from_js(cmd_cfg_to_parse.get(FixCoreCommandsRoot), CustomCommandsConfig) - except Exception as e: - log.error(f"Can not parse command templates. Fall back to defaults. Reason: {e}", exc_info=e) + try: + command_config_json = command_templates or CustomCommandsConfig().json() + migrated_commands = migrate_command_config(command_config_json) + cmd_cfg_to_parse = migrated_commands or command_config_json + command_overrides = (get_overrides(FixCoreCommandsConfigId) or {}).get(FixCoreCommandsRoot) + if command_overrides: + cmd_cfg_to_parse = cast( + Json, + merge_json_elements(cmd_cfg_to_parse, {FixCoreCommandsRoot: command_overrides}), + ) + commands_config = from_js(cmd_cfg_to_parse.get(FixCoreCommandsRoot), CustomCommandsConfig) + except Exception as e: + log.error(f"Can not parse command templates. Fall back to defaults. Reason: {e}", exc_info=e) snapshots_config = SnapshotsScheduleConfig() - if snapshot_schedule: - try: - snapshots_config = from_js(snapshot_schedule.get(FixCoreSnapshotsRoot), SnapshotsScheduleConfig) - except Exception as e: - log.error(f"Can not parse snapshot schedule. Fall back to defaults. Reason: {e}", exc_info=e) + try: + snapshot_config_json = snapshot_schedule or SnapshotsScheduleConfig().json() + snapshot_overrides = (get_overrides(FixCoreSnapshotsConfigId) or {}).get(FixCoreSnapshotsRoot) + if snapshot_overrides: + snapshot_config_json = cast( + Json, + merge_json_elements(snapshot_config_json, {FixCoreSnapshotsRoot: snapshot_overrides}), + ) + snapshots_config = from_js(snapshot_config_json.get(FixCoreSnapshotsRoot), SnapshotsScheduleConfig) + except Exception as e: + log.error(f"Can not parse snapshot schedule. Fall back to defaults. Reason: {e}", exc_info=e) return CoreConfig( api=ed.api, @@ -818,7 +830,7 @@ def migrate_command_config(cmd_config: Json) -> Optional[Json]: def config_from_db( args: Namespace, db: StandardDatabase, - get_core_overrides: Callable[[], Optional[Json]], + get_overrides: Callable[[ConfigId], Optional[Json]], collection_name: str = "configs", ) -> CoreConfig: if configs := db.collection(collection_name) if db.has_collection(collection_name) else None: @@ -830,5 +842,5 @@ def config_from_db( snapshots_config_entity = cast(Optional[Json], configs.get(FixCoreSnapshotsConfigId)) snapshots_config = snapshots_config_entity.get("config") if snapshots_config_entity else None - return parse_config(args, config, get_core_overrides, command_config, snapshots_config) - return parse_config(args, {}, get_core_overrides) + return parse_config(args, config, get_overrides, command_config, snapshots_config) + return parse_config(args, {}, get_overrides) diff --git a/fixcore/fixcore/system_start.py b/fixcore/fixcore/system_start.py index 479e32b66a..b0e722d21e 100644 --- a/fixcore/fixcore/system_start.py +++ b/fixcore/fixcore/system_start.py @@ -243,7 +243,7 @@ def key_value(kv: str) -> Tuple[str, JsonElement]: def empty_config(args: Optional[List[str]] = None) -> CoreConfig: - return parse_config(parse_args(args or []), {}, lambda: None) + return parse_config(parse_args(args or []), {}, lambda _: None) # Note: this method should be called from every started process as early as possible diff --git a/fixcore/tests/fixcore/config/config_handler_service_test.py b/fixcore/tests/fixcore/config/config_handler_service_test.py index 0ed9d56905..27a3e189d8 100644 --- a/fixcore/tests/fixcore/config/config_handler_service_test.py +++ b/fixcore/tests/fixcore/config/config_handler_service_test.py @@ -8,6 +8,7 @@ from fixcore.analytics import InMemoryEventSender from fixcore.config import ConfigHandler, ConfigEntity, ConfigValidation, ConfigOverride from fixcore.config.config_handler_service import ConfigHandlerService +from fixcore.core_config import CustomCommandsConfig, FixCoreCommandsConfigId from fixcore.ids import ConfigId from fixcore.message_bus import CoreMessage, Event, Message from tests.fixcore.message_bus_test import wait_for_message @@ -300,6 +301,37 @@ async def test_config_yaml(config_handler: ConfigHandler, config_model: List[Kin assert expect_override_comment in config_with_override_yaml +@pytest.mark.asyncio +async def test_config_yaml_handles_non_string_list_overrides_for_core_commands( + config_handler: ConfigHandler, +) -> None: + override = { + "custom_commands": { + "commands": [ + { + "name": "search-foo", + "info": "Search Foo resources.", + "template": "search is(foo)", + } + ] + } + } + + await config_handler.put_config( + ConfigEntity(FixCoreCommandsConfigId, CustomCommandsConfig().json()), validate=False + ) + cast(ConfigHandlerService, config_handler).override_service = cast( + ConfigOverride, + SimpleNamespace(get_override=lambda cfg_id: override if cfg_id == FixCoreCommandsConfigId else None), + ) + + config_yaml = await config_handler.config_yaml(FixCoreCommandsConfigId) + + assert config_yaml is not None + assert "custom_commands:" in config_yaml + assert "commands:" in config_yaml + + @pytest.mark.asyncio async def test_config_change_emits_event(config_handler: ConfigHandler, all_events: List[Message]) -> None: # Put a config diff --git a/fixcore/tests/fixcore/core_config_test.py b/fixcore/tests/fixcore/core_config_test.py index 73ead04288..69f3f650a3 100644 --- a/fixcore/tests/fixcore/core_config_test.py +++ b/fixcore/tests/fixcore/core_config_test.py @@ -16,9 +16,13 @@ migrate_core_config, migrate_command_config, CustomCommandsConfig, + SnapshotsScheduleConfig, alias_templates, + FixCoreCommandsConfigId, FixCoreCommandsRoot, FixCoreConfigId, + FixCoreSnapshotsConfigId, + FixCoreSnapshotsRoot, current_git_hash, ) from fixcore.system_start import parse_args @@ -40,7 +44,7 @@ def test_parse_broken(config_json: Json) -> None: del cfg["fixcore"]["api"]["https_port"] # parse this configuration - parsed = parse_config(parse_args(["--analytics-opt-out"]), cfg, lambda: None) + parsed = parse_config(parse_args(["--analytics-opt-out"]), cfg, lambda _: None) parsed_json = to_js(parsed.editable, strip_attr="kind") # web_hosts and https_port were not available and are reverted to the default values @@ -57,13 +61,13 @@ def test_parse_broken(config_json: Json) -> None: def test_read_config(config_json: Json) -> None: - parsed = parse_config(parse_args(["--analytics-opt-out"]), config_json, lambda: None) + parsed = parse_config(parse_args(["--analytics-opt-out"]), config_json, lambda _: None) assert parsed.json() == config_json def test_override_via_cmd_line(default_config: CoreConfig) -> None: config = {"runtime": {"debug": False}} - parsed = parse_config(parse_args(["--debug"]), config, lambda: None) + parsed = parse_config(parse_args(["--debug"]), config, lambda _: None) assert parsed.runtime.debug == True @@ -101,12 +105,57 @@ def test_config_override(config_json: Json) -> None: parsed = parse_config( parse_args(["--analytics-opt-out"]), cfg, - lambda: overrides.get(FixCoreConfigId), + lambda config_id: overrides.get(config_id), ) assert parsed.api.web_hosts == ["11.12.13.14"] assert parsed.api.https_port == 1337 +def test_command_config_override() -> None: + overrides = { + FixCoreCommandsConfigId: { + FixCoreCommandsRoot: { + "commands": [ + { + "name": "compliance_orphan_disk_match", + "info": "Read-only selector for disks without an attached consumer signal.", + "template": "search is(volume) with (empty, <-- is(instance))", + } + ] + } + } + } + + parsed = parse_config( + parse_args(["--analytics-opt-out"]), + {}, + lambda config_id: overrides.get(config_id), + ) + + assert any(cmd.name == "compliance_orphan_disk_match" for cmd in parsed.custom_commands.commands) + + +def test_snapshot_config_override() -> None: + overrides = { + FixCoreSnapshotsConfigId: { + FixCoreSnapshotsRoot: { + "snapshots": { + "weekly": {"schedule": "0 12 * * 1", "retain": 9}, + } + } + } + } + + parsed = parse_config( + parse_args(["--analytics-opt-out"]), + {}, + lambda config_id: overrides.get(config_id), + ) + + assert parsed.snapshots.snapshots["weekly"].schedule == "0 12 * * 1" + assert parsed.snapshots.snapshots["weekly"].retain == 9 + + def test_model() -> None: model = config_model() assert {m["fqn"] for m in model} == {