From 9135c6a18cefe76234fc583cbcc6fef00f89aaa5 Mon Sep 17 00:00:00 2001 From: "Ian C." <108159253+ic-dev21@users.noreply.github.com> Date: Thu, 5 Feb 2026 19:28:55 -0500 Subject: [PATCH 1/7] Hilo_state corruption logic Add a recreation in case of corruption. --- pyhilo/const.py | 2 +- pyhilo/util/state.py | 36 +++++++++++++++++++++++++++++++----- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/pyhilo/const.py b/pyhilo/const.py index f1a7870..3ab1571 100755 --- a/pyhilo/const.py +++ b/pyhilo/const.py @@ -1,7 +1,7 @@ import logging import platform -import uuid from typing import Final +import uuid import aiohttp diff --git a/pyhilo/util/state.py b/pyhilo/util/state.py index 11d8e3c..5b3789a 100644 --- a/pyhilo/util/state.py +++ b/pyhilo/util/state.py @@ -128,11 +128,37 @@ async def get_state(state_yaml: str) -> StateDict: state_yaml ): # noqa: PTH113 - isfile is fine and simpler in this case. return _get_defaults(StateDict) # type: ignore - async with aiofiles.open(state_yaml, mode="r") as yaml_file: - LOG.debug("Loading state from yaml") - content = await yaml_file.read() - state_yaml_payload: StateDict = yaml.safe_load(content) - return state_yaml_payload + + try: + async with aiofiles.open(state_yaml, mode="r") as yaml_file: + LOG.debug("Loading state from yaml") + content = await yaml_file.read() + state_yaml_payload: StateDict = yaml.safe_load(content) + + # Handle corrupted/empty YAML files + if state_yaml_payload is None: + LOG.warning( + "State file %s is corrupted or empty, reinitializing with defaults", + state_yaml, + ) + defaults = _get_defaults(StateDict) # type: ignore + async with aiofiles.open(state_yaml, mode="w") as yaml_file_write: + content = yaml.dump(defaults) + await yaml_file_write.write(content) + return defaults + + return state_yaml_payload + except yaml.YAMLError as e: + LOG.error( + "Failed to parse state file %s: %s. Reinitializing with defaults.", + state_yaml, + e, + ) + defaults = _get_defaults(StateDict) # type: ignore + async with aiofiles.open(state_yaml, mode="w") as yaml_file_write: + content = yaml.dump(defaults) + await yaml_file_write.write(content) + return defaults async def set_state( From 786626620dcd843deaef89e63f64ea3cace275ef Mon Sep 17 00:00:00 2001 From: "Ian C." <108159253+ic-dev21@users.noreply.github.com> Date: Sun, 8 Feb 2026 11:51:11 -0500 Subject: [PATCH 2/7] Mypy linting Removed a few type: ignore and properly typed the rest. --- pyhilo/util/state.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyhilo/util/state.py b/pyhilo/util/state.py index bb2a9fb..1df2c2e 100644 --- a/pyhilo/util/state.py +++ b/pyhilo/util/state.py @@ -77,7 +77,7 @@ class StateDict(TypedDict, total=False): T = TypeVar("T", bound="StateDict") -def _get_defaults(cls: type[T]) -> dict[str, Any]: +def _get_defaults(cls: type[T]) -> T: """Generate a default dict based on typed dict This function recursively creates a nested dictionary structure that mirrors @@ -127,13 +127,13 @@ async def get_state(state_yaml: str) -> StateDict: if not isfile( state_yaml ): # noqa: PTH113 - isfile is fine and simpler in this case. - return _get_defaults(StateDict) # type: ignore + return _get_defaults(StateDict) try: async with aiofiles.open(state_yaml, mode="r") as yaml_file: LOG.debug("Loading state from yaml") content = await yaml_file.read() - state_yaml_payload: StateDict = yaml.safe_load(content) + state_yaml_payload: StateDict | None = yaml.safe_load(content) # Handle corrupted/empty YAML files if state_yaml_payload is None: @@ -141,7 +141,7 @@ async def get_state(state_yaml: str) -> StateDict: "State file %s is corrupted or empty, reinitializing with defaults", state_yaml, ) - defaults = _get_defaults(StateDict) # type: ignore + defaults = _get_defaults(StateDict) async with aiofiles.open(state_yaml, mode="w") as yaml_file_write: content = yaml.dump(defaults) await yaml_file_write.write(content) @@ -154,7 +154,7 @@ async def get_state(state_yaml: str) -> StateDict: state_yaml, e, ) - defaults = _get_defaults(StateDict) # type: ignore + defaults = _get_defaults(StateDict) async with aiofiles.open(state_yaml, mode="w") as yaml_file_write: content = yaml.dump(defaults) await yaml_file_write.write(content) From eb9a67306552fe9c86a0947860c8faef5dbb4ab0 Mon Sep 17 00:00:00 2001 From: "Ian C." <108159253+ic-dev21@users.noreply.github.com> Date: Sat, 14 Feb 2026 14:04:53 -0500 Subject: [PATCH 3/7] Update state.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ajout d'un check supplémentaire si juste un string au lieu d'un YAML malformé. --- pyhilo/util/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyhilo/util/state.py b/pyhilo/util/state.py index 1df2c2e..9a84d0b 100644 --- a/pyhilo/util/state.py +++ b/pyhilo/util/state.py @@ -136,7 +136,7 @@ async def get_state(state_yaml: str) -> StateDict: state_yaml_payload: StateDict | None = yaml.safe_load(content) # Handle corrupted/empty YAML files - if state_yaml_payload is None: + if state_yaml_payload is None or not isinstance(state_yaml_payload, dict): LOG.warning( "State file %s is corrupted or empty, reinitializing with defaults", state_yaml, From 416eee8efaa4a942758188b73cabf5b97bc40d6a Mon Sep 17 00:00:00 2001 From: "Ian C." <108159253+ic-dev21@users.noreply.github.com> Date: Tue, 17 Feb 2026 20:25:38 -0500 Subject: [PATCH 4/7] Update state.py Fix pourpossible race condition --- pyhilo/util/state.py | 76 +++++++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 29 deletions(-) diff --git a/pyhilo/util/state.py b/pyhilo/util/state.py index 9a84d0b..1974b9f 100644 --- a/pyhilo/util/state.py +++ b/pyhilo/util/state.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio +import os +import tempfile from datetime import datetime from os.path import isfile from typing import Any, ForwardRef, TypedDict, TypeVar, get_type_hints @@ -116,38 +118,53 @@ def _get_defaults(cls: type[T]) -> T: new_dict[k] = None # type: ignore[literal-required] return new_dict # type: ignore[return-value] +def _write_state(state_yaml: str, state: dict[str, Any]) -> None: + "Write state atomically to a temp file, this prevents reading a file being written to" -async def get_state(state_yaml: str) -> StateDict: + dir_name = os.path.dirname(os.path.abspath(state_yaml)) + content = yaml.dump(state) + with tempfile.NamedTemporaryFile(mode = "w", dir=dir_name, delete=False, suffix=".tmp") as tmp: + tmp.write(content) + tmp_path = tmp.name + os.replace(tmp_path, state_yaml) + + +async def get_state(state_yaml: str, _already_locked: bool = False) -> StateDict: """Read in state yaml. :param state_yaml: filename where to read the state :type state_yaml: ``str`` + :param _already_locked: Whether the lock is already held by the caller (e.g. set_state). + Prevents deadlock when corruption recovery needs to write defaults. + :type _already_locked: ``bool`` :rtype: ``StateDict`` """ - if not isfile( - state_yaml - ): # noqa: PTH113 - isfile is fine and simpler in this case. + if not isfile(state_yaml): # noqa: PTH113 - isfile is fine and simpler in this case. return _get_defaults(StateDict) try: async with aiofiles.open(state_yaml, mode="r") as yaml_file: LOG.debug("Loading state from yaml") content = await yaml_file.read() - state_yaml_payload: StateDict | None = yaml.safe_load(content) - - # Handle corrupted/empty YAML files - if state_yaml_payload is None or not isinstance(state_yaml_payload, dict): - LOG.warning( - "State file %s is corrupted or empty, reinitializing with defaults", - state_yaml, - ) - defaults = _get_defaults(StateDict) - async with aiofiles.open(state_yaml, mode="w") as yaml_file_write: - content = yaml.dump(defaults) - await yaml_file_write.write(content) - return defaults + + state_yaml_payload: StateDict | None = yaml.safe_load(content) + + # Handle corrupted/empty YAML files + if state_yaml_payload is None or not isinstance(state_yaml_payload, dict): + LOG.warning( + "State file %s is corrupted or empty, reinitializing with defaults", + state_yaml, + ) + defaults = _get_defaults(StateDict) + if _already_locked: + _write_state(state_yaml, defaults) + else: + async with lock: + _write_state(state_yaml, defaults) + return defaults return state_yaml_payload + except yaml.YAMLError as e: LOG.error( "Failed to parse state file %s: %s. Reinitializing with defaults.", @@ -155,12 +172,15 @@ async def get_state(state_yaml: str) -> StateDict: e, ) defaults = _get_defaults(StateDict) - async with aiofiles.open(state_yaml, mode="w") as yaml_file_write: - content = yaml.dump(defaults) - await yaml_file_write.write(content) + if _already_locked: + _write_state(state_yaml, defaults) + else: + async with lock: + _write_state(state_yaml, defaults) return defaults + async def set_state( state_yaml: str, key: str, @@ -169,6 +189,7 @@ async def set_state( ), ) -> None: """Save state yaml. + :param state_yaml: filename where to read the state :type state_yaml: ``str`` :param key: Key name @@ -178,14 +199,11 @@ async def set_state( :rtype: ``StateDict`` """ async with lock: # note ic-dev21: on lock le fichier pour être sûr de finir la job - current_state = await get_state(state_yaml) or {} + current_state = await get_state(state_yaml, _already_locked=True) or {} merged_state: dict[str, Any] = {key: {**current_state.get(key, {}), **state}} # type: ignore[dict-item] new_state: dict[str, Any] = {**current_state, **merged_state} - async with aiofiles.open(state_yaml, mode="w") as yaml_file: - LOG.debug("Saving state to yaml file") - # TODO: Use asyncio.get_running_loop() and run_in_executor to write - # to the file in a non blocking manner. Currently, the file writes - # are properly async but the yaml dump is done synchronously on the - # main event loop. - content = yaml.dump(new_state) - await yaml_file.write(content) + LOG.debug("Saving state to yaml file") + # TODO: Use asyncio.get_running_loop() and run_in_executor to write + # to the file in a non blocking manner. Currently, yaml.dump is + # synchronous on the main event loop. + _write_state(state_yaml, new_state) \ No newline at end of file From 584d6e39cb69cbac8cdd09eb108fbec281c55e09 Mon Sep 17 00:00:00 2001 From: "Ian C." <108159253+ic-dev21@users.noreply.github.com> Date: Tue, 17 Feb 2026 20:30:52 -0500 Subject: [PATCH 5/7] Update state.py Linting --- pyhilo/util/state.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/pyhilo/util/state.py b/pyhilo/util/state.py index 1974b9f..f013f26 100644 --- a/pyhilo/util/state.py +++ b/pyhilo/util/state.py @@ -3,10 +3,10 @@ from __future__ import annotations import asyncio -import os -import tempfile from datetime import datetime +import os from os.path import isfile +import tempfile from typing import Any, ForwardRef, TypedDict, TypeVar, get_type_hints import aiofiles @@ -118,12 +118,15 @@ def _get_defaults(cls: type[T]) -> T: new_dict[k] = None # type: ignore[literal-required] return new_dict # type: ignore[return-value] -def _write_state(state_yaml: str, state: dict[str, Any]) -> None: + +def _write_state(state_yaml: str, state: dict[str, Any] | StateDict) -> None: "Write state atomically to a temp file, this prevents reading a file being written to" dir_name = os.path.dirname(os.path.abspath(state_yaml)) content = yaml.dump(state) - with tempfile.NamedTemporaryFile(mode = "w", dir=dir_name, delete=False, suffix=".tmp") as tmp: + with tempfile.NamedTemporaryFile( + mode="w", dir=dir_name, delete=False, suffix=".tmp" + ) as tmp: tmp.write(content) tmp_path = tmp.name os.replace(tmp_path, state_yaml) @@ -139,7 +142,9 @@ async def get_state(state_yaml: str, _already_locked: bool = False) -> StateDict :type _already_locked: ``bool`` :rtype: ``StateDict`` """ - if not isfile(state_yaml): # noqa: PTH113 - isfile is fine and simpler in this case. + if not isfile( + state_yaml + ): # noqa: PTH113 - isfile is fine and simpler in this case. return _get_defaults(StateDict) try: @@ -180,7 +185,6 @@ async def get_state(state_yaml: str, _already_locked: bool = False) -> StateDict return defaults - async def set_state( state_yaml: str, key: str, @@ -206,4 +210,4 @@ async def set_state( # TODO: Use asyncio.get_running_loop() and run_in_executor to write # to the file in a non blocking manner. Currently, yaml.dump is # synchronous on the main event loop. - _write_state(state_yaml, new_state) \ No newline at end of file + _write_state(state_yaml, new_state) From 584dd3319cfcd349ead258db661739eaa6bef70b Mon Sep 17 00:00:00 2001 From: "Ian C." <108159253+ic-dev21@users.noreply.github.com> Date: Tue, 17 Feb 2026 20:40:43 -0500 Subject: [PATCH 6/7] Ajout chmod MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parce que pouvoir voir le fichier sans être root j'pense que c'est raisonnable. --- pyhilo/util/state.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyhilo/util/state.py b/pyhilo/util/state.py index f013f26..0f180dd 100644 --- a/pyhilo/util/state.py +++ b/pyhilo/util/state.py @@ -129,6 +129,7 @@ def _write_state(state_yaml: str, state: dict[str, Any] | StateDict) -> None: ) as tmp: tmp.write(content) tmp_path = tmp.name + os.chmod(tmp_path, 0o644) os.replace(tmp_path, state_yaml) From c97f9a2fe8e1a35fdcafd57da861ba149a75aa38 Mon Sep 17 00:00:00 2001 From: "Ian C." <108159253+ic-dev21@users.noreply.github.com> Date: Fri, 20 Feb 2026 20:40:00 -0500 Subject: [PATCH 7/7] F-string logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Un petit gain d'efficacité tant qu'à y être --- pyhilo/api.py | 12 ++++++------ pyhilo/device/__init__.py | 10 +++++----- pyhilo/devices.py | 2 +- pyhilo/websocket.py | 18 ++++++++++-------- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/pyhilo/api.py b/pyhilo/api.py index 553743e..cdbc452 100755 --- a/pyhilo/api.py +++ b/pyhilo/api.py @@ -291,7 +291,7 @@ async def _async_request( try: data = await resp.json(content_type=None) except json.decoder.JSONDecodeError: - LOG.warning(f"JSON Decode error: {resp.__dict__}") + LOG.warning("JSON Decode error: %s", resp.__dict__) message = await resp.text() data = {"error": message} else: @@ -353,7 +353,7 @@ async def _async_handle_on_backoff(self, _: dict[str, Any]) -> None: err: ClientResponseError = err_info[1].with_traceback(err_info[2]) # type: ignore if err.status in (401, 403): - LOG.warning(f"Refreshing websocket token {err.request_info.url}") + LOG.warning("Refreshing websocket token %s", err.request_info.url) if ( "client/negotiate" in str(err.request_info.url) and err.request_info.method == "POST" @@ -361,7 +361,7 @@ async def _async_handle_on_backoff(self, _: dict[str, Any]) -> None: LOG.info( "401 detected on websocket, refreshing websocket token. Old url: {self.ws_url} Old Token: {self.ws_token}" ) - LOG.info(f"401 detected on {err.request_info.url}") + LOG.info("401 detected on %s", err.request_info.url) async with self._backoff_refresh_lock_ws: await self.refresh_ws_token() await self.get_websocket_params() @@ -480,7 +480,7 @@ async def fb_install(self, fb_id: str) -> None: json=body, ) except ClientResponseError as err: - LOG.error(f"ClientResponseError: {err}") + LOG.error("ClientResponseError: %s", err) if err.status in (401, 403): raise InvalidCredentialsError("Invalid credentials") from err raise RequestError(err) from err @@ -518,14 +518,14 @@ async def android_register(self) -> None: data=parsed_body, ) except ClientResponseError as err: - LOG.error(f"ClientResponseError: {err}") + LOG.error("ClientResponseError: %s", err) if err.status in (401, 403): raise InvalidCredentialsError("Invalid credentials") from err raise RequestError(err) from err LOG.debug("Android client register: %s", resp) msg: str = resp.get("message", "") if msg.startswith("Error="): - LOG.error(f"Android registration error: {msg}") + LOG.error("Android registration error: %s", msg) raise RequestError token = msg.split("=")[-1] LOG.debug("Calling set_state android_register") diff --git a/pyhilo/device/__init__.py b/pyhilo/device/__init__.py index c92095f..026b5b2 100644 --- a/pyhilo/device/__init__.py +++ b/pyhilo/device/__init__.py @@ -51,7 +51,7 @@ def __init__( def update(self, **kwargs: Dict[str, Union[str, int, Dict]]) -> None: # TODO(dvd): This has to be re-written, this is not dynamic at all. if self._api.log_traces: - LOG.debug(f"[TRACE] Adding device {kwargs}") + LOG.debug("[TRACE] Adding device %s", kwargs) for orig_att, val in kwargs.items(): att = camel_to_snake(orig_att) if reading_att := HILO_READING_TYPES.get(orig_att): @@ -70,7 +70,7 @@ def update(self, **kwargs: Dict[str, Union[str, int, Dict]]) -> None: self.update_readings(DeviceReading(**reading)) # type: ignore if att not in HILO_DEVICE_ATTRIBUTES: - LOG.warning(f"Unknown device attribute {att}: {val}") + LOG.warning("Unknown device attribute %s: %s", att, val) continue elif att in HILO_LIST_ATTRIBUTES: # This is where we generated the supported_attributes and settable_attributes @@ -108,7 +108,7 @@ def update(self, **kwargs: Dict[str, Union[str, int, Dict]]) -> None: async def set_attribute(self, attribute: str, value: Union[str, int, None]) -> None: if dev_attribute := cast(DeviceAttribute, self._api.dev_atts(attribute)): - LOG.debug(f"{self._tag} Setting {dev_attribute} to {value}") + LOG.debug("%s Setting %s to %s", self._tag, dev_attribute, value) await self._set_attribute(dev_attribute, value) return LOG.warning( @@ -134,7 +134,7 @@ async def _set_attribute( ) ) else: - LOG.warning(f"{self._tag} Invalid attribute {attribute} for device") + LOG.warning("%s Invalid attribute %s for device", self._tag, attribute) def get_attribute(self, attribute: str) -> Union[DeviceReading, None]: if dev_attribute := cast(DeviceAttribute, self._api.dev_atts(attribute)): @@ -245,7 +245,7 @@ def __init__(self, **kwargs: Dict[str, Any]): else "" ) if not self.device_attribute: - LOG.warning(f"Received invalid reading for {self.device_id}: {kwargs}") + LOG.warning("Received invalid reading for %s: %s", self.device_id, kwargs) def __repr__(self) -> str: return f"" diff --git a/pyhilo/devices.py b/pyhilo/devices.py index c9de57c..e927bb4 100644 --- a/pyhilo/devices.py +++ b/pyhilo/devices.py @@ -87,7 +87,7 @@ def generate_device(self, device: dict) -> HiloDevice: try: device_type = HILO_DEVICE_TYPES[dev.type] except KeyError: - LOG.warning(f"Unknown device type {dev.type}, adding as Sensor") + LOG.warning("Unknown device type %s, adding as Sensor", dev.type) device_type = "Sensor" dev.__class__ = globals()[device_type] return dev diff --git a/pyhilo/websocket.py b/pyhilo/websocket.py index 4522f73..b6d69c9 100755 --- a/pyhilo/websocket.py +++ b/pyhilo/websocket.py @@ -173,7 +173,9 @@ async def _async_receive_json(self) -> list[Dict[str, Any]]: response = await self._client.receive(300) if response.type in (WSMsgType.CLOSE, WSMsgType.CLOSED, WSMsgType.CLOSING): - LOG.error(f"Websocket: Received event to close connection: {response.type}") + LOG.error( + "Websocket: Received event to close connection: %s", response.type + ) raise ConnectionClosedError("Connection was closed.") if response.type == WSMsgType.ERROR: @@ -183,7 +185,7 @@ async def _async_receive_json(self) -> list[Dict[str, Any]]: raise ConnectionFailedError if response.type != WSMsgType.TEXT: - LOG.error(f"Websocket: Received invalid message: {response}") + LOG.error("Websocket: Received invalid message: %s", response) raise InvalidMessageError(f"Received non-text message: {response.type}") messages: list[Dict[str, Any]] = [] @@ -196,7 +198,7 @@ async def _async_receive_json(self) -> list[Dict[str, Any]]: except ValueError as v_exc: raise InvalidMessageError("Received invalid JSON") from v_exc except json.decoder.JSONDecodeError as j_exc: - LOG.error(f"Received invalid JSON: {msg}") + LOG.error("Received invalid JSON: %s", msg) LOG.exception(j_exc) data = {} @@ -307,14 +309,14 @@ async def async_connect(self) -> None: **proxy_env, ) except (ClientError, ServerDisconnectedError, WSServerHandshakeError) as err: - LOG.error(f"Unable to connect to WS server {err}") + LOG.error("Unable to connect to WS server %s", err) if hasattr(err, "status") and err.status in (401, 403, 404, 409): raise InvalidCredentialsError("Invalid credentials") from err except Exception as err: - LOG.error(f"Unable to connect to WS server {err}") + LOG.error("Unable to connect to WS server %s", err) raise CannotConnectError(err) from err - LOG.info(f"Connected to websocket server {self._api.endpoint}") + LOG.info("Connected to websocket server %s", self._api.endpoint) # Quick pause to prevent race condition await asyncio.sleep(0.05) @@ -353,11 +355,11 @@ async def async_listen(self) -> None: LOG.info("Websocket: Listen cancelled.") raise except ConnectionClosedError as err: - LOG.error(f"Websocket: Closed while listening: {err}") + LOG.error("Websocket: Closed while listening: %s", err) LOG.exception(err) pass except InvalidMessageError as err: - LOG.warning(f"Websocket: Received invalid json : {err}") + LOG.warning("Websocket: Received invalid json : %s", err) pass finally: LOG.info("Websocket: Listen completed; cleaning up")