diff --git a/CHANGELOG.md b/CHANGELOG.md index 293db8b2..fc3ee40e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,29 @@ Changelogs prior to v2.0 is pruned, but was available in the v2.x releases This project should adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), though for pre-releases PEP 440 takes precedence. +## [Unreleased] + +### Breaking Changes + +* The icalendar dependency is updated from 6 to 7 - not because 3.0 depends on icalendar7, but because I'm planning to use icalendar7-features in some upcoming 3.x. If this causes problems for you, just reach out and I will downgrade the dependency, release a new 3.0.1, and possibly procrastinate the icalendar7-stuff until 4.0. + +### Added + +* **Stalwart CalDAV server** added to Docker test server framework. +* **Async multiget fallback in `_async_load()`** -- `CalendarObjectResource._async_load()` now has the same two-stage fallback as sync `load()`: on a 404, first tries calendar-multiget REPORT, then re-fetches by UID. Previously the async path only had the UID re-fetch fallback. + +### Fixed + +* Fixed two bugs in `DAVClient` rate-limit auto-detection: (1) `is_supported('rate-limit', dict)` returns `{'enable': False}` (a truthy dict) for unconfigured clients, causing rate-limiting to be enabled unintentionally; (2) crash (`TypeError: '>' not supported between 'int' and 'NoneType'`) when `rate_limit_max_sleep` is `None` and a 429 is received. +* Fixed `AsyncDAVClient` rate-limit handling to be fully symmetric with the sync client: same `Optional[bool]` default, same features-based auto-detect, same adaptive backoff accumulating sleep time across retries. +* Fixed `Principal._async_get_property()` override having an incompatible signature (missing `use_cached` and `**passthrough`) and reimplementing PROPFIND logic already handled correctly by the parent `DAVObject._async_get_property()`. The override has been removed. +* Fixed inconsistent URL quoting for calendar object UIDs containing slashes -- both `_generate_url()` and `_find_id_and_path()` in `calendarobject_ops.py` now share a single `_quote_uid()` helper (related to https://github.com/python-caldav/caldav/issues/143). +* Fixed `expand_simple_props()` return value handling. + +### Test Framework + +* Added async rate-limit unit tests matching the sync test suite. + ## [3.0.0a2] - 2026-02-25 (Alpha Release) **This is an alpha release for testing purposes.** Please report issues at https://github.com/python-caldav/caldav/issues diff --git a/caldav/async_davclient.py b/caldav/async_davclient.py index 9712ffe8..7fa9f6ef 100644 --- a/caldav/async_davclient.py +++ b/caldav/async_davclient.py @@ -34,6 +34,17 @@ _H2_AVAILABLE = True except ImportError: pass + + class _HttpxBearerAuth(httpx.Auth): + """httpx-compatible bearer token auth.""" + + def __init__(self, password: str) -> None: + self.password = password + + def auth_flow(self, request): + request.headers["Authorization"] = f"Bearer {self.password}" + yield request + except ImportError: pass @@ -144,7 +155,7 @@ def __init__( features: FeatureSet | dict | str | None = None, enable_rfc6764: bool = True, require_tls: bool = True, - rate_limit_handle: bool = False, + rate_limit_handle: Optional[bool] = None, rate_limit_default_sleep: Optional[int] = None, rate_limit_max_sleep: Optional[int] = None, ) -> None: @@ -167,7 +178,8 @@ def __init__( enable_rfc6764: Enable RFC6764 DNS-based service discovery. require_tls: Require TLS for discovered services (security consideration). rate_limit_handle: When True, automatically sleep and retry on 429/503 - responses. When False (default), raise RateLimitError immediately. + responses. When None (default), auto-detected from server features. + When False, raise RateLimitError immediately. rate_limit_default_sleep: Fallback sleep seconds when the server's 429 response omits a Retry-After header. None (default) means raise rather than sleeping when no Retry-After is provided. @@ -237,6 +249,8 @@ def __init__( # Use explicit None check to preserve empty strings (needed for servers with no auth) self.username = username if username is not None else url_username self.password = password if password is not None else url_password + # Strip credentials from stored URL to avoid leaking them in log messages + self.url = self.url.unauth() # Setup authentication self.auth = auth @@ -258,6 +272,16 @@ def __init__( } self.headers.update(headers) + rate_limit = self.features.is_supported("rate-limit", dict) + if rate_limit_handle is None: + if rate_limit and rate_limit.get("enable"): + rate_limit_handle = True + if "default_sleep" in rate_limit: + rate_limit_default_sleep = rate_limit["default_sleep"] + if "max_sleep" in rate_limit: + rate_limit_max_sleep = rate_limit["max_sleep"] + else: + rate_limit_handle = False self.rate_limit_handle = rate_limit_handle self.rate_limit_default_sleep = rate_limit_default_sleep self.rate_limit_max_sleep = rate_limit_max_sleep @@ -339,13 +363,16 @@ async def request( method: str = "GET", body: str = "", headers: Mapping[str, str] | None = None, + rate_limit_time_slept: float = 0, ) -> AsyncDAVResponse: """ Send an async HTTP request, with optional rate-limit sleep-and-retry. Catches RateLimitError from _async_request. When rate_limit_handle is - True and a usable sleep duration is available, sleeps then retries once. - Otherwise re-raises immediately. + True and a usable sleep duration is available, sleeps then retries with + adaptive backoff (each retry adds half the already-slept time). Stops + retrying when rate_limit_max_sleep is exceeded or no sleep duration is + available. Otherwise re-raises immediately. """ try: return await self._async_request(url, method, body, headers) @@ -357,10 +384,17 @@ async def request( self.rate_limit_default_sleep, self.rate_limit_max_sleep, ) - if sleep_seconds is None: + if rate_limit_time_slept: + sleep_seconds += rate_limit_time_slept / 2 + if sleep_seconds is None or ( + self.rate_limit_max_sleep is not None + and rate_limit_time_slept > self.rate_limit_max_sleep + ): raise await asyncio.sleep(sleep_seconds) - return await self._async_request(url, method, body, headers) + return await self.request( + url, method, body, headers, rate_limit_time_slept + sleep_seconds + ) async def _async_request( self, @@ -846,7 +880,10 @@ def build_auth_object(self, auth_types: list[str] | None = None) -> None: # Build auth object - use appropriate classes for httpx or niquests if auth_type == "bearer": - self.auth = HTTPBearerAuth(self.password) + if _USE_HTTPX: + self.auth = _HttpxBearerAuth(self.password) + else: + self.auth = HTTPBearerAuth(self.password) elif auth_type == "digest": if _USE_HTTPX: self.auth = httpx.DigestAuth(self.username, self.password) @@ -930,12 +967,20 @@ async def get_calendars(self, principal: Optional["Principal"] = None) -> list[" from caldav.operations.calendarset_ops import ( _extract_calendars_from_propfind_results as extract_calendars, ) + from caldav.operations.principal_ops import ( + _extract_calendar_home_set_from_results as extract_home_set, + ) if principal is None: principal = await self.get_principal() # Get calendar-home-set from principal - calendar_home_url = await self._get_calendar_home_set(principal) + response = await self.propfind( + str(principal.url), + props=self.CALENDAR_HOME_SET_PROPS, + depth=0, + ) + calendar_home_url = extract_home_set(response.results) if not calendar_home_url: return [] @@ -958,73 +1003,6 @@ async def get_calendars(self, principal: Optional["Principal"] = None) -> list[" for info in calendar_infos ] - async def _get_calendar_home_set(self, principal: "Principal") -> str | None: - """Get the calendar-home-set URL for a principal. - - Args: - principal: Principal object - - Returns: - Calendar home set URL or None - """ - from caldav.operations.principal_ops import ( - _extract_calendar_home_set_from_results as extract_home_set, - ) - - # Try to get from principal properties - response = await self.propfind( - str(principal.url), - props=self.CALENDAR_HOME_SET_PROPS, - depth=0, - ) - - return extract_home_set(response.results) - - async def get_events( - self, - calendar: "Calendar", - start: Any | None = None, - end: Any | None = None, - ) -> list["Event"]: - """Get events from a calendar. - - This is a convenience method that searches for VEVENT objects in the - calendar, optionally filtered by date range. - - Args: - calendar: Calendar to search - start: Start of date range (optional) - end: End of date range (optional) - - Returns: - List of Event objects. - - Example: - from datetime import datetime - events = await client.get_events( - calendar, - start=datetime(2024, 1, 1), - end=datetime(2024, 12, 31) - ) - """ - return await self.search_calendar(calendar, event=True, start=start, end=end) - - async def get_todos( - self, - calendar: "Calendar", - include_completed: bool = False, - ) -> list["Todo"]: - """Get todos from a calendar. - - Args: - calendar: Calendar to search - include_completed: Whether to include completed todos - - Returns: - List of Todo objects. - """ - return await self.search_calendar(calendar, todo=True, include_completed=include_completed) - async def search_calendar( self, calendar: "Calendar", @@ -1109,50 +1087,11 @@ async def search_principals(self, name: str | None = None) -> list["Principal"]: Raises: ReportError: If the server doesn't support principal search """ - from lxml import etree - - from caldav.collection import CalendarSet, Principal - from caldav.elements import cdav, dav - - if name: - name_filter = [ - dav.PropertySearch() + [dav.Prop() + [dav.DisplayName()]] + dav.Match(value=name) - ] - else: - name_filter = [] - - query = ( - dav.PrincipalPropertySearch() - + name_filter - + [dav.Prop(), cdav.CalendarHomeSet(), dav.DisplayName()] - ) - response = await self.report(str(self.url), etree.tostring(query.xmlelement())) - + body = self._build_principal_search_query(name) + response = await self.report(str(self.url), body) if response.status >= 300: raise error.ReportError(f"{response.status} {response.reason} - {response.raw}") - - principal_dict = response._find_objects_and_props() - ret = [] - for x in principal_dict: - p = principal_dict[x] - if dav.DisplayName.tag not in p: - continue - pname = p[dav.DisplayName.tag].text - error.assert_(not p[dav.DisplayName.tag].getchildren()) - error.assert_(not p[dav.DisplayName.tag].items()) - chs = p[cdav.CalendarHomeSet.tag] - error.assert_(not chs.items()) - error.assert_(not chs.text) - chs_href = chs.getchildren() - error.assert_(len(chs_href) == 1) - error.assert_(not chs_href[0].items()) - error.assert_(not chs_href[0].getchildren()) - chs_url = chs_href[0].text - calendar_home_set = CalendarSet(client=self, url=chs_url) - ret.append( - Principal(client=self, url=x, name=pname, calendar_home_set=calendar_home_set) - ) - return ret + return self._parse_principal_search_response(response._find_objects_and_props()) async def principals(self, name: str | None = None) -> list["Principal"]: """ diff --git a/caldav/base_client.py b/caldav/base_client.py index 43cc378c..57e47fe0 100644 --- a/caldav/base_client.py +++ b/caldav/base_client.py @@ -193,6 +193,67 @@ def _raise_authorization_error(self, url_str: str, reason_source: Any) -> NoRetu reason = "None given" raise error.AuthorizationError(url=url_str, reason=reason) + def _build_principal_search_query(self, name: str | None) -> bytes: + """Build the XML body for a principal-property-search REPORT.""" + from lxml import etree + + from caldav.elements import cdav, dav + + name_filter = ( + [dav.PropertySearch() + [dav.Prop() + [dav.DisplayName()]] + dav.Match(value=name)] + if name + else [] + ) + query = ( + dav.PrincipalPropertySearch() + + name_filter + + [dav.Prop(), cdav.CalendarHomeSet(), dav.DisplayName()] + ) + return etree.tostring(query.xmlelement()) + + def _parse_principal_search_response(self, principal_dict: dict) -> list: + """Parse principal-property-search REPORT results into Principal objects.""" + from caldav.collection import CalendarSet, Principal + from caldav.elements import cdav, dav + + ret = [] + for x in principal_dict: + p = principal_dict[x] + if dav.DisplayName.tag not in p: + continue + name = p[dav.DisplayName.tag].text + error.assert_(not p[dav.DisplayName.tag].getchildren()) + error.assert_(not p[dav.DisplayName.tag].items()) + chs = p[cdav.CalendarHomeSet.tag] + error.assert_(not chs.items()) + error.assert_(not chs.text) + chs_href = chs.getchildren() + error.assert_(len(chs_href) == 1) + error.assert_(not chs_href[0].items()) + error.assert_(not chs_href[0].getchildren()) + chs_url = chs_href[0].text + calendar_home_set = CalendarSet(client=self, url=chs_url) + ret.append( + Principal(client=self, url=x, name=name, calendar_home_set=calendar_home_set) + ) + return ret + + def get_events(self, calendar: Any, start: Any = None, end: Any = None) -> Any: + """Get events from a calendar, optionally filtered by date range. + + For sync clients returns a list directly. + For async clients returns a coroutine that must be awaited. + """ + return self.search_calendar(calendar, event=True, start=start, end=end) + + def get_todos(self, calendar: Any, include_completed: bool = False) -> Any: + """Get todos from a calendar. + + For sync clients returns a list directly. + For async clients returns a coroutine that must be awaited. + """ + return self.search_calendar(calendar, todo=True, include_completed=include_completed) + @abstractmethod def build_auth_object(self, auth_types: list[str] | None = None) -> None: """ diff --git a/caldav/calendarobjectresource.py b/caldav/calendarobjectresource.py index 7632f606..fe211cad 100644 --- a/caldav/calendarobjectresource.py +++ b/caldav/calendarobjectresource.py @@ -18,7 +18,7 @@ from collections import defaultdict from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, Optional -from urllib.parse import ParseResult, SplitResult, quote +from urllib.parse import ParseResult, SplitResult import icalendar from dateutil.rrule import rrulestr @@ -53,6 +53,7 @@ from .lib.error import errmsg from .lib.python_utilities import to_normal_str, to_unicode, to_wire from .lib.url import URL +from .operations.calendarobject_ops import _quote_uid log = logging.getLogger("caldav") @@ -752,23 +753,28 @@ async def _async_load(self, only_if_unloaded: bool = False) -> Self: raise error.NotFoundError(errmsg(r)) self.data = r.raw # type: ignore except error.NotFoundError: - # Fallback: re-fetch by UID (server may have changed the URL) uid = self.id - if uid and self.parent and hasattr(self.parent, "get_object_by_uid"): + if uid: + # Fallback 1: try multiget (REPORT may work even when GET fails) try: - obj = await self.parent.get_object_by_uid(uid) - if obj: - self.url = obj.url - self.data = obj.data - if hasattr(obj, "props"): - self.props.update(obj.props) - return self - except error.NotFoundError: + return await self._async_load_by_multiget() + except Exception: pass + # Fallback 2: re-fetch by UID (server may have changed the URL) + if self.parent and hasattr(self.parent, "get_object_by_uid"): + try: + obj = await self.parent.get_object_by_uid(uid) + if obj: + self.url = obj.url + self.data = obj.data + if hasattr(obj, "props"): + self.props.update(obj.props) + return self + except error.NotFoundError: + pass raise except Exception: - # Note: load_by_multiget is sync-only, not supported in async mode yet - raise + return await self._async_load_by_multiget() if "Etag" in r.headers: self.props[dav.GetEtag.tag] = r.headers["Etag"] @@ -776,6 +782,17 @@ async def _async_load(self, only_if_unloaded: bool = False) -> Self: self.props[cdav.ScheduleTag.tag] = r.headers["Schedule-Tag"] return self + async def _async_load_by_multiget(self) -> Self: + """Async implementation of load_by_multiget.""" + error.assert_(self.url) + items = await self.parent._async_multiget(event_urls=[self.url], raise_notfound=True) + if not items: + raise error.NotFoundError(self.url) + _url, self.data = items[0] + error.assert_(self.data) + error.assert_(len(items) == 1) + return self + def load_by_multiget(self) -> Self: """ Some servers do not accept a GET, but we can still do a REPORT @@ -783,14 +800,14 @@ def load_by_multiget(self) -> Self: """ error.assert_(self.url) mydata = self.parent._multiget(event_urls=[self.url], raise_notfound=True) - try: - url, self.data = next(mydata) - except StopIteration: + url_data = next(mydata, None) + if url_data is None: ## We shouldn't come here. Something is wrong. ## TODO: research it ## As of 2025-05-20, this code section is used by ## TestForServerECloud::testCreateOverwriteDeleteEvent raise error.NotFoundError(self.url) + url, self.data = url_data error.assert_(self.data) error.assert_(next(mydata, None) is None) return self @@ -897,7 +914,7 @@ def _generate_url(self): ## See https://github.com/python-caldav/caldav/issues/143 for the rationale behind double-quoting slashes ## TODO: should try to wrap my head around issues that arises when id contains weird characters. maybe it's ## better to generate a new uuid here, particularly if id is in some unexpected format. - url = self.parent.url.join(quote(self.id.replace("/", "%2F")) + ".ics") + url = self.parent.url.join(_quote_uid(self.id) + ".ics") assert " " not in str(url) return url @@ -1245,7 +1262,7 @@ def _set_deprecated_vobject_instance(self, inst: "vobject.base.Component"): DeprecationWarning, stacklevel=2, ) - return self._get_vobject_instance(inst) + return self._set_vobject_instance(inst) vobject_instance: "vobject.base.VBase" = property( _get_vobject_instance, diff --git a/caldav/collection.py b/caldav/collection.py index 1134c228..2d60e019 100644 --- a/caldav/collection.py +++ b/caldav/collection.py @@ -58,6 +58,18 @@ class CalendarSet(DAVObject): A CalendarSet is a set of calendars. """ + def _calendars_from_results(self, results) -> list["Calendar"]: + """Convert PropfindResult list into Calendar objects.""" + from caldav.operations.calendarset_ops import ( + _extract_calendars_from_propfind_results, + ) + + calendar_infos = _extract_calendars_from_propfind_results(results) + return [ + Calendar(client=self.client, url=info.url, name=info.name, id=info.cal_id, parent=self) + for info in calendar_infos + ] + def get_calendars(self) -> list["Calendar"]: """ List all calendar collections in this set. @@ -74,69 +86,20 @@ def get_calendars(self) -> list["Calendar"]: Example (async): calendars = await calendar_set.get_calendars() """ - # Delegate to client for dual-mode support if self.is_async_client: return self._async_calendars() - cals = [] - data = self.children(cdav.Calendar.tag) - for c_url, c_type, c_name in data: - try: - cal_id = c_url.split("/")[-2] - if not cal_id: - continue - except Exception: - log.error(f"Calendar {c_name} has unexpected url {c_url}") - cal_id = None - cals.append(Calendar(self.client, id=cal_id, url=c_url, parent=self, name=c_name)) - - return cals - - async def _async_calendars(self) -> list["Calendar"]: - """Async implementation of calendars() using the client.""" - from caldav.operations.base import _is_calendar_resource as is_calendar_resource - from caldav.operations.calendarset_ops import ( - _extract_calendar_id_from_url as extract_calendar_id_from_url, + response = self.client.propfind( + str(self.url), props=self.client.CALENDAR_LIST_PROPS, depth=1 ) + return self._calendars_from_results(response.results) - # Fetch calendars via PROPFIND + async def _async_calendars(self) -> list["Calendar"]: + """Async implementation of get_calendars().""" response = await self.client.propfind( - str(self.url), - props=[ - "{DAV:}resourcetype", - "{DAV:}displayname", - "{urn:ietf:params:xml:ns:caldav}supported-calendar-component-set", - "{http://apple.com/ns/ical/}calendar-color", - "{http://calendarserver.org/ns/}getctag", - ], - depth=1, + str(self.url), props=self.client.CALENDAR_LIST_PROPS, depth=1 ) - - # Process results to extract calendars - calendars = [] - for result in response.results or []: - # Check if this is a calendar resource - if not is_calendar_resource(result.properties): - continue - - # Extract calendar info - url = result.href - name = result.properties.get("{DAV:}displayname") - cal_id = extract_calendar_id_from_url(url) - - if not cal_id: - continue - - cal = Calendar( - client=self.client, - url=url, - name=name, - id=cal_id, - parent=self, - ) - calendars.append(cal) - - return calendars + return self._calendars_from_results(response.results) def calendars(self) -> list["Calendar"]: """ @@ -349,24 +312,6 @@ async def create( return principal - async def _async_get_property(self, prop): - """Async version of get_property for use with async clients.""" - if self.url is None: - raise ValueError("Unexpected value None for self.url") - - response = await self.client.propfind( - str(self.url), - props=[prop.tag if hasattr(prop, "tag") else str(prop)], - depth=0, - ) - - if response.results: - for result in response.results: - value = result.properties.get(prop.tag if hasattr(prop, "tag") else str(prop)) - if value is not None: - return value - return None - def make_calendar( self, name: str | None = None, @@ -1038,6 +983,24 @@ def multiget(self, event_urls: Iterable[URL], raise_notfound: bool = False) -> I parent=self, ) + async def _async_multiget( + self, event_urls: Iterable[URL], raise_notfound: bool = False + ) -> list[tuple[str, str]]: + """Async version of _multiget — returns a list of (url, data) tuples.""" + if self.url is None: + raise ValueError("Unexpected value None for self.url") + + prop = dav.Prop() + cdav.CalendarData() + root = cdav.CalendarMultiGet() + prop + [dav.Href(value=u.path) for u in event_urls] + response = await self._async_query(root, None, "report") + results = response.expand_simple_props([cdav.CalendarData()]) + if raise_notfound: + for href in response.statuses: + status = response.statuses[href] + if status and "404" in status: + raise error.NotFoundError(f"Status {status} in {href}") + return [(r, results[r][cdav.CalendarData.tag]) for r in results] + def calendar_multiget(self, *largs, **kwargs): """ get multiple events' data @@ -1653,7 +1616,7 @@ def _generate_fake_sync_token(self, objects: list["CalendarObjectResource"]) -> etags.append(str(obj.url.canonical())) etags.sort() ## Consistent ordering combined = "|".join(etags) - hash_value = hashlib.md5(combined.encode()).hexdigest() + hash_value = hashlib.md5(combined.encode(), usedforsecurity=False).hexdigest() return f"fake-{hash_value}" def get_objects_by_sync_token( diff --git a/caldav/compatibility_hints.py b/caldav/compatibility_hints.py index 4de73bdf..3eae4521 100644 --- a/caldav/compatibility_hints.py +++ b/caldav/compatibility_hints.py @@ -84,10 +84,12 @@ class FeatureSet: "description": "Principal has one or more calendars. Some servers and providers comes with a pre-defined calendar for each user, for other servers a calendar has to be explicitly created (supported means there exists a calendar - it may be because the calendar was already provisioned together with the principal, or it may be because a calendar was created manually, the checks can't see the difference)"}, "rate-limit": { "type": "client-feature", - "description": "client (or test code) must not send requests too fast", + "description": "client (or test code) must sleep a bit between requests. Pro-active rate limiting is done through interval and count, server-flagged rate-limiting is controlled through default_sleep/max_sleep", "extra_keys": { "interval": "Rate limiting window, in seconds", "count": "Max number of requests to send within the interval", + "max_sleep": "Max sleep when hitting a 429 or 503 with retry-after, in seconds", + "default_sleep": "Sleep for this long when hitting a 429, in seconds" }}, "search-cache": { "type": "server-peculiarity", @@ -462,7 +464,7 @@ def _default(self, feature_info): elif feature_type in ('tests-behaviour', 'client-hints'): return { } else: - breakpoint() + raise ValueError(f"Unknown feature type: {feature_type!r}") def is_supported(self, feature, return_type=bool, return_defaults=True, accept_fragile=False): """Work in progress @@ -936,9 +938,11 @@ def dotted_feature_set_list(self, compact=False): ## TODO: this applies only to test runs, not to ordinary usage 'rate-limit': { 'enable': True, - 'interval': 10, + 'interval': 2, 'count': 1, - 'description': "It's needed to manually empty trashbin frequently when running tests. Since this oepration takes some time and/or there are some caches, it's needed to run tests slowly, even when hammering the 'empty thrashbin' frequently", + 'default_sleep': 4, + 'max_sleep': 120, + 'description': "It's needed to manually empty trashbin frequently when running tests. Since this operation takes some time and/or there are some caches, it's needed to run tests slowly, even when hammering the 'empty thrashbin' frequently", }, 'auto-connect.url': { 'basepath': '/remote.php/dav', @@ -951,8 +955,7 @@ def dotted_feature_set_list(self, compact=False): zimbra = { 'auto-connect.url': {'basepath': '/dav/'}, 'delete-calendar': {'support': 'fragile', 'behaviour': 'may move to trashbin instead of deleting immediately'}, - ## save-load.get-by-url was unsupported in older Zimbra versions (GET to - ## valid calendar object URLs returned 404), but works in zimbra/zcs-foss:latest + 'save-load.get-by-url': {'support': 'fragile', 'behaviour': '404 most of the time - but sometimes 200. Weird, should be investigated more'}, ## Zimbra treats same-UID events across calendars as aliases of the same event 'save.duplicate-uid.cross-calendar': {'support': 'unsupported'}, 'search.recurrences.expanded.exception': {'support': 'unsupported'}, ## TODO: verify @@ -1024,22 +1027,10 @@ def dotted_feature_set_list(self, compact=False): "search.time-range.todo": { "support": "unsupported" }, - "search.text.case-sensitive": { - "support": "unsupported" - }, - "search.text.case-insensitive": { - "support": "unsupported" - }, - "search.text.category": { - "support": "ungraceful" - }, + "search.text": False, ## sometimes ungraceful "search.is-not-defined": { "support": "fragile" }, - "search.text.by-uid": { - "support": "fragile", - "behaviour": "sometimes the text search delivers everything, other times it doesn't deliver anything. When the text search delivers everything, then the post-filtering will save the day" - }, "search.recurrences.includes-implicit": { "support": "unsupported", "behaviour": "cannot reliably test due to broken comp-type filtering" @@ -1362,6 +1353,22 @@ def dotted_feature_set_list(self, compact=False): ], } +## Stalwart - all-in-one mail & collaboration server (CalDAV added 2024/2025) +## https://stalw.art/ +## CalDAV served at /dav/cal// over HTTP on port 8080. +## Feature support mostly unknown until tested; starting with empty hints. +stalwart = { + 'rate-limit': { + 'enable': True, + 'default_sleep': 3, + 'max_sleep': 60 + }, + 'create-calendar.auto': True, + 'principal-search': {'support': 'ungraceful'}, + 'search.recurrences.expanded.exception': False, + 'search.time-range.alarm': False, +} + ## Lots of transient problems with purelymail purelymail = { ## Purelymail claims that the search indexes are "lazily" populated, @@ -1411,8 +1418,10 @@ def dotted_feature_set_list(self, compact=False): }, 'rate-limit': { 'enable': True, - 'interval': 4, + 'interval': 1, 'count': 1, + 'default_sleep': 4, + 'max_sleep': 30 }, 'search.comp-type-optional': {'support': 'fragile', 'description': 'unexpected results from date-search without comp-type - but only sometimes - TODO: research more'}, 'search.recurrences.expanded': {'support': 'unsupported'}, diff --git a/caldav/config.py b/caldav/config.py index 25d0819d..a566f750 100644 --- a/caldav/config.py +++ b/caldav/config.py @@ -88,7 +88,7 @@ def read_config(fn, interactive_error=False): cfg = read_config(config_file) if cfg: return cfg - return None + return {} ## This can probably be refactored into fewer lines ... try: diff --git a/caldav/davclient.py b/caldav/davclient.py index 808a1bc4..61578083 100644 --- a/caldav/davclient.py +++ b/caldav/davclient.py @@ -211,7 +211,7 @@ def __init__( features: FeatureSet | dict | str = None, enable_rfc6764: bool = True, require_tls: bool = True, - rate_limit_handle: bool = False, + rate_limit_handle: Optional[bool] = None, rate_limit_default_sleep: Optional[int] = None, rate_limit_max_sleep: Optional[int] = None, ) -> None: @@ -358,6 +358,16 @@ def __init__( self._principal = None + rate_limit = self.features.is_supported("rate-limit", dict) + if rate_limit_handle is None: + if rate_limit and rate_limit.get("enable"): + rate_limit_handle = True + if "default_sleep" in rate_limit: + rate_limit_default_sleep = rate_limit["default_sleep"] + if "max_sleep" in rate_limit: + rate_limit_max_sleep = rate_limit["max_sleep"] + else: + rate_limit_handle = False self.rate_limit_handle = rate_limit_handle self.rate_limit_default_sleep = rate_limit_default_sleep self.rate_limit_max_sleep = rate_limit_max_sleep @@ -382,7 +392,7 @@ def __exit__( if hasattr(self, "teardown"): try: self.teardown() - except: + except TypeError: self.teardown(self) def close(self) -> None: @@ -409,48 +419,14 @@ def search_principals(self, name=None): Raises: ReportError: If the server doesn't support principal search """ - if name: - name_filter = [ - dav.PropertySearch() + [dav.Prop() + [dav.DisplayName()]] + dav.Match(value=name) - ] - else: - name_filter = [] - - query = ( - dav.PrincipalPropertySearch() - + name_filter - + [dav.Prop(), cdav.CalendarHomeSet(), dav.DisplayName()] - ) - response = self.report(self.url, etree.tostring(query.xmlelement())) - + body = self._build_principal_search_query(name) + response = self.report(self.url, body) ## Possibly we should follow redirects (response status 3xx), but as ## for now we're just treating it in the same way as 4xx and 5xx - ## probably the server did not support the operation if response.status >= 300: raise error.ReportError(f"{response.status} {response.reason} - {response.raw}") - - principal_dict = response._find_objects_and_props() - ret = [] - for x in principal_dict: - p = principal_dict[x] - if dav.DisplayName.tag not in p: - continue - name = p[dav.DisplayName.tag].text - error.assert_(not p[dav.DisplayName.tag].getchildren()) - error.assert_(not p[dav.DisplayName.tag].items()) - chs = p[cdav.CalendarHomeSet.tag] - error.assert_(not chs.items()) - error.assert_(not chs.text) - chs_href = chs.getchildren() - error.assert_(len(chs_href) == 1) - error.assert_(not chs_href[0].items()) - error.assert_(not chs_href[0].getchildren()) - chs_url = chs_href[0].text - calendar_home_set = CalendarSet(client=self, url=chs_url) - ret.append( - Principal(client=self, url=x, name=name, calendar_home_set=calendar_home_set) - ) - return ret + return self._parse_principal_search_response(response._find_objects_and_props()) def principals(self, name=None): """ @@ -533,12 +509,20 @@ def get_calendars(self, principal: Principal | None = None) -> list[Calendar]: from caldav.operations.calendarset_ops import ( _extract_calendars_from_propfind_results as extract_calendars, ) + from caldav.operations.principal_ops import ( + _extract_calendar_home_set_from_results as extract_home_set, + ) if principal is None: principal = self.principal() # Get calendar-home-set from principal - calendar_home_url = self._get_calendar_home_set(principal) + response = self.propfind( + str(principal.url), + props=self.CALENDAR_HOME_SET_PROPS, + depth=0, + ) + calendar_home_url = extract_home_set(response.results) if not calendar_home_url: # Fall back to the principal URL as calendar home # (some servers like GMX don't support calendar-home-set) @@ -563,73 +547,6 @@ def get_calendars(self, principal: Principal | None = None) -> list[Calendar]: for info in calendar_infos ] - def _get_calendar_home_set(self, principal: Principal) -> str | None: - """Get the calendar-home-set URL for a principal. - - Args: - principal: Principal object - - Returns: - Calendar home set URL or None - """ - from caldav.operations.principal_ops import ( - _extract_calendar_home_set_from_results as extract_home_set, - ) - - # Try to get from principal properties - response = self.propfind( - str(principal.url), - props=self.CALENDAR_HOME_SET_PROPS, - depth=0, - ) - - return extract_home_set(response.results) - - def get_events( - self, - calendar: Calendar, - start: Any | None = None, - end: Any | None = None, - ) -> list["Event"]: - """Get events from a calendar. - - This is a convenience method that searches for VEVENT objects in the - calendar, optionally filtered by date range. - - Args: - calendar: Calendar to search - start: Start of date range (optional) - end: End of date range (optional) - - Returns: - List of Event objects. - - Example: - from datetime import datetime - events = client.get_events( - calendar, - start=datetime(2024, 1, 1), - end=datetime(2024, 12, 31) - ) - """ - return self.search_calendar(calendar, event=True, start=start, end=end) - - def get_todos( - self, - calendar: Calendar, - include_completed: bool = False, - ) -> list["Todo"]: - """Get todos from a calendar. - - Args: - calendar: Calendar to search - include_completed: Whether to include completed todos - - Returns: - List of Todo objects. - """ - return self.search_calendar(calendar, todo=True, include_completed=include_completed) - def search_calendar( self, calendar: Calendar, @@ -694,7 +611,7 @@ def check_dav_support(self) -> str | None: ## element that should come with caldav extras. ## Anyway, packing this into a try-except in case it fails. response = self.options(self.principal().url) - except: + except Exception: response = self.options(str(self.url)) return response.headers.get("DAV", None) @@ -939,6 +856,7 @@ def request( method: str = "GET", body: str = "", headers: Mapping[str, str] = None, + rate_limit_time_slept=0, ) -> DAVResponse: """ Send a generic HTTP request. @@ -964,10 +882,15 @@ def request( self.rate_limit_default_sleep, self.rate_limit_max_sleep, ) - if sleep_seconds is None: + if rate_limit_time_slept: + sleep_seconds += rate_limit_time_slept / 2 + if sleep_seconds is None or ( + self.rate_limit_max_sleep is not None + and rate_limit_time_slept > self.rate_limit_max_sleep + ): raise time.sleep(sleep_seconds) - return self._sync_request(url, method, body, headers) + return self.request(url, method, body, headers, rate_limit_time_slept + sleep_seconds) def _sync_request( self, diff --git a/caldav/davobject.py b/caldav/davobject.py index 2d4cf018..d43352c3 100644 --- a/caldav/davobject.py +++ b/caldav/davobject.py @@ -96,7 +96,8 @@ def __init__( self.url = None else: self.url = URL.objectify(url) - assert " " not in str(self.url) + if self.url is not None and " " in str(self.url): + raise ValueError(f"URL must not contain spaces: {self.url!r}") @property def canonical_url(self) -> str: @@ -170,6 +171,26 @@ def children(self, type: str | None = None) -> list[tuple[URL, Any, Any]]: ## the properties we've already fetched return c + def _build_xml_body(self, root) -> bytes | str: + """Serialize a DAV element (or raw bytes/str) to a request body.""" + if root: + if hasattr(root, "xmlelement"): + return etree.tostring( + root.xmlelement(), + encoding="utf-8", + xml_declaration=True, + pretty_print=error.debug_dump_communication, + ) + return root + return "" + + def _build_propfind_root(self, props): + """Build a Propfind XML element from a list of props, or return None.""" + if props is not None and len(props) > 0: + prop = dav.Prop() + props + return dav.Propfind() + prop + return None + def _query_properties(self, props: Sequence[BaseElement] | None = None, depth: int = 0): """ This is an internal method for doing a propfind query. It's a @@ -181,25 +202,13 @@ def _query_properties(self, props: Sequence[BaseElement] | None = None, depth: i if self.is_async_client: return self._async_query_properties(props, depth) - root = None - # build the propfind request - if props is not None and len(props) > 0: - prop = dav.Prop() + props - root = dav.Propfind() + prop - - return self._query(root, depth) + return self._query(self._build_propfind_root(props), depth) async def _async_query_properties( self, props: Sequence[BaseElement] | None = None, depth: int = 0 ): """Async implementation of _query_properties.""" - root = None - # build the propfind request - if props is not None and len(props) > 0: - prop = dav.Prop() + props - root = dav.Propfind() + prop - - return await self._async_query(root, depth) + return await self._async_query(self._build_propfind_root(props), depth) def _query( self, @@ -219,17 +228,7 @@ def _query( if self.is_async_client: return self._async_query(root, depth, query_method, url, expected_return_value) - body = "" - if root: - if hasattr(root, "xmlelement"): - body = etree.tostring( - root.xmlelement(), - encoding="utf-8", - xml_declaration=True, - pretty_print=error.debug_dump_communication, - ) - else: - body = root + body = self._build_xml_body(root) if url is None: url = self.url ret = getattr(self.client, query_method)(url, body, depth) @@ -256,17 +255,7 @@ async def _async_query( expected_return_value=None, ): """Async implementation of _query.""" - body = "" - if root: - if hasattr(root, "xmlelement"): - body = etree.tostring( - root.xmlelement(), - encoding="utf-8", - xml_declaration=True, - pretty_print=error.debug_dump_communication, - ) - else: - body = root + body = self._build_xml_body(root) if url is None: url = self.url ret = await getattr(self.client, query_method)(url, body, depth) @@ -492,18 +481,7 @@ def set_properties(self, props: Any | None = None) -> Self: prop = dav.Prop() + props set_elem = dav.Set() + prop root = dav.PropertyUpdate() + set_elem - - body = "" - if root: - if hasattr(root, "xmlelement"): - body = etree.tostring( - root.xmlelement(), - encoding="utf-8", - xml_declaration=True, - pretty_print=error.debug_dump_communication, - ) - else: - body = root + body = self._build_xml_body(root) if self.url is None: raise ValueError("Unexpected value None for self.url") @@ -523,18 +501,7 @@ async def _async_set_properties(self, props: Any | None = None) -> Self: prop = dav.Prop() + props set_elem = dav.Set() + prop root = dav.PropertyUpdate() + set_elem - - body = "" - if root: - if hasattr(root, "xmlelement"): - body = etree.tostring( - root.xmlelement(), - encoding="utf-8", - xml_declaration=True, - pretty_print=error.debug_dump_communication, - ) - else: - body = root + body = self._build_xml_body(root) if self.url is None: raise ValueError("Unexpected value None for self.url") diff --git a/caldav/operations/calendar_ops.py b/caldav/operations/calendar_ops.py index 2bdbfdd2..07630b46 100644 --- a/caldav/operations/calendar_ops.py +++ b/caldav/operations/calendar_ops.py @@ -129,7 +129,7 @@ def _generate_fake_sync_token(etags_and_urls: list[tuple[str | None, str]]) -> s parts.sort() # Consistent ordering combined = "|".join(parts) - hash_value = hashlib.md5(combined.encode()).hexdigest() + hash_value = hashlib.md5(combined.encode(), usedforsecurity=False).hexdigest() return f"fake-{hash_value}" diff --git a/caldav/operations/calendarobject_ops.py b/caldav/operations/calendarobject_ops.py index c31de227..fa7ff54f 100644 --- a/caldav/operations/calendarobject_ops.py +++ b/caldav/operations/calendarobject_ops.py @@ -42,7 +42,17 @@ class CalendarObjectData: def _generate_uid() -> str: """Generate a new UID for a calendar object.""" - return str(uuid.uuid1()) + return str(uuid.uuid4()) + + +def _quote_uid(uid: str) -> str: + """ + URL-quote a UID for use in a CalDAV object URL. + + Slashes are double-quoted (replaced with %2F before percent-encoding) + per https://github.com/python-caldav/caldav/issues/143. + """ + return quote(uid.replace("/", "%2F")) def _generate_url(parent_url: str, uid: str) -> str: @@ -58,8 +68,7 @@ def _generate_url(parent_url: str, uid: str) -> str: Returns: Full URL for the calendar object """ - # Double-quote slashes per https://github.com/python-caldav/caldav/issues/143 - quoted_uid = quote(uid.replace("/", "%2F")) + quoted_uid = _quote_uid(uid) if not parent_url.endswith("/"): parent_url += "/" return f"{parent_url}{quoted_uid}.ics" @@ -135,7 +144,7 @@ def _find_id_and_path( if given_path: path = given_path else: - path = quote(uid.replace("/", "%2F")) + ".ics" + path = _quote_uid(uid) + ".ics" return uid, path diff --git a/caldav/protocol/__init__.py b/caldav/protocol/__init__.py index 74c89eaa..ddf68acc 100644 --- a/caldav/protocol/__init__.py +++ b/caldav/protocol/__init__.py @@ -17,7 +17,6 @@ """ from .types import ( - CalendarInfo, CalendarQueryResult, DAVMethod, DAVRequest, @@ -36,7 +35,6 @@ "DAVRequest", "DAVResponse", # Result types - "CalendarInfo", "CalendarQueryResult", "MultiGetResult", "MultistatusResponse", diff --git a/caldav/protocol/types.py b/caldav/protocol/types.py index 457db79d..13697e1a 100644 --- a/caldav/protocol/types.py +++ b/caldav/protocol/types.py @@ -219,25 +219,3 @@ class PrincipalInfo: calendar_home_set: str | None = None displayname: str | None = None calendar_user_address_set: list[str] = field(default_factory=list) - - -@dataclass -class CalendarInfo: - """ - Information about a calendar collection. - - Attributes: - url: Calendar URL - displayname: Display name - description: Calendar description - color: Calendar color (vendor extension) - supported_components: List of supported component types (VEVENT, VTODO, etc.) - ctag: Calendar CTag for change detection - """ - - url: str - displayname: str | None = None - description: str | None = None - color: str | None = None - supported_components: list[str] = field(default_factory=list) - ctag: str | None = None diff --git a/caldav/protocol/xml_builders.py b/caldav/protocol/xml_builders.py index f744d1eb..6009ca83 100644 --- a/caldav/protocol/xml_builders.py +++ b/caldav/protocol/xml_builders.py @@ -235,25 +235,6 @@ def _build_sync_collection_body( return etree.tostring(sync_collection.xmlelement(), encoding="utf-8", xml_declaration=True) -def _build_freebusy_query_body( - start: datetime, - end: datetime, -) -> bytes: - """ - Build free-busy-query REPORT request body. - - Args: - start: Start of free-busy period - end: End of free-busy period - - Returns: - UTF-8 encoded XML bytes - """ - root = cdav.FreeBusyQuery() + [cdav.TimeRange(start, end)] - - return etree.tostring(root.xmlelement(), encoding="utf-8", xml_declaration=True) - - def _build_mkcalendar_body( displayname: str | None = None, description: str | None = None, @@ -298,39 +279,6 @@ def _build_mkcalendar_body( return etree.tostring(mkcalendar.xmlelement(), encoding="utf-8", xml_declaration=True) -def _build_mkcol_body( - displayname: str | None = None, - resource_types: list[BaseElement] | None = None, -) -> bytes: - """ - Build MKCOL (extended) request body. - - Args: - displayname: Collection display name - resource_types: List of resource type elements - - Returns: - UTF-8 encoded XML bytes - """ - prop = dav.Prop() - - if displayname: - prop += dav.DisplayName(displayname) - - if resource_types: - rt = dav.ResourceType() - for rt_elem in resource_types: - rt += rt_elem - prop += rt - else: - prop += dav.ResourceType() + dav.Collection() - - set_elem = dav.Set() + prop - mkcol = dav.Mkcol() + set_elem - - return etree.tostring(mkcol.xmlelement(), encoding="utf-8", xml_declaration=True) - - # Property name to element mapping @@ -396,33 +344,3 @@ def _prop_name_to_element(name: str, value: Any | None = None) -> BaseElement | return cls() return None - - -def _to_utc_date_string(ts: datetime) -> str: - """ - Convert datetime to UTC date string for CalDAV. - - Args: - ts: datetime object (may or may not have timezone) - - Returns: - UTC date string in format YYYYMMDDTHHMMSSZ - """ - from datetime import timezone - - utc_tz = timezone.utc - - if ts.tzinfo is None: - # Assume local time, convert to UTC - try: - ts = ts.astimezone(utc_tz) - except Exception: - # For very old Python versions or edge cases - import tzlocal - - ts = ts.replace(tzinfo=tzlocal.get_localzone()) - ts = ts.astimezone(utc_tz) - else: - ts = ts.astimezone(utc_tz) - - return ts.strftime("%Y%m%dT%H%M%SZ") diff --git a/caldav/protocol/xml_parsers.py b/caldav/protocol/xml_parsers.py index 6f099f11..c14452ec 100644 --- a/caldav/protocol/xml_parsers.py +++ b/caldav/protocol/xml_parsers.py @@ -257,12 +257,31 @@ def _parse_calendar_multiget_response( Returns: List of CalendarQueryResult with calendar data """ - return parse_calendar_query_response(body, status_code, huge_tree) + return _parse_calendar_query_response(body, status_code, huge_tree) # Helper functions +def _normalize_href(text: str) -> str: + """ + Normalize an href string from a DAV response element. + + Handles the Confluence double-encoding bug (%2540 → %40) and converts + absolute URLs to path-only strings so callers always work with paths. + """ + # Fix for https://github.com/python-caldav/caldav/issues/471 + # Confluence server quotes the user email twice. + if "%2540" in text: + text = text.replace("%2540", "%40") + href = unquote(text) + # Ref https://github.com/python-caldav/caldav/issues/435 + # Some servers return absolute URLs; convert to path. + if ":" in href: + href = unquote(URL(href).path) + return href + + def _strip_to_multistatus(tree: _Element) -> _Element | list[_Element]: """ Strip outer elements to get to the multistatus content. @@ -301,14 +320,7 @@ def _parse_response_element( status = elem.text _validate_status(status) elif elem.tag == dav.Href.tag: - # Fix for double-encoded URLs (e.g., Confluence) - text = elem.text or "" - if "%2540" in text: - text = text.replace("%2540", "%40") - href = unquote(text) - # Convert absolute URLs to paths - if ":" in href: - href = unquote(URL(href).path) + href = _normalize_href(elem.text or "") elif elem.tag == dav.PropStat.tag: propstats.append(elem) diff --git a/caldav/response.py b/caldav/response.py index bd2feef9..901f1223 100644 --- a/caldav/response.py +++ b/caldav/response.py @@ -9,7 +9,6 @@ import warnings from collections.abc import Iterable from typing import TYPE_CHECKING, Any, cast -from urllib.parse import unquote from lxml import etree from lxml.etree import _Element @@ -18,7 +17,13 @@ from caldav.elements.base import BaseElement from caldav.lib import error from caldav.lib.python_utilities import to_normal_str -from caldav.lib.url import URL +from caldav.protocol.xml_parsers import ( + _normalize_href, + _validate_status, +) +from caldav.protocol.xml_parsers import ( + _strip_to_multistatus as _proto_strip, +) if TYPE_CHECKING: # Protocol for HTTP response objects (works with httpx, niquests, requests) @@ -165,12 +170,7 @@ def _strip_to_multistatus(self) -> _Element | list[_Element]: (The equivalent of this method could probably be found with a simple XPath query, but I'm not much into XPath) """ - tree = self.tree - if tree.tag == "xml" and tree[0].tag == dav.MultiStatus.tag: - return tree[0] - if tree.tag == dav.MultiStatus.tag: - return self.tree - return [self.tree] + return _proto_strip(self.tree) def validate_status(self, status: str) -> None: """ @@ -181,13 +181,7 @@ def validate_status(self, status: str) -> None: makes sense to me, but I've only seen it from SOGo, and it's not in accordance with the examples in rfc6578. """ - if ( - " 200 " not in status - and " 201 " not in status - and " 207 " not in status - and " 404 " not in status - ): - raise error.ResponseError(status) + _validate_status(status) def _parse_response(self, response: _Element) -> tuple[str, list[_Element], Any | None]: """ @@ -208,11 +202,7 @@ def _parse_response(self, response: _Element) -> tuple[str, list[_Element], Any self.validate_status(status) elif elem.tag == dav.Href.tag: assert not href - # Fix for https://github.com/python-caldav/caldav/issues/471 - # Confluence server quotes the user email twice. We unquote it manually. - if "%2540" in elem.text: - elem.text = elem.text.replace("%2540", "%40") - href = unquote(elem.text) + href = _normalize_href(elem.text or "") elif elem.tag == dav.PropStat.tag: propstats.append(elem) elif elem.tag == "{DAV:}error": @@ -232,14 +222,10 @@ def _parse_response(self, response: _Element) -> tuple[str, list[_Element], Any error.assert_(href) if check_404: error.assert_("404" in status) - ## TODO: is this safe/sane? - ## Ref https://github.com/python-caldav/caldav/issues/435 the paths returned may be absolute URLs, - ## but the caller expects them to be paths. Could we have issues when a server has same path - ## but different URLs for different elements? Perhaps href should always be made into an URL-object? - if ":" in href: - href = unquote(URL(href).path) return (cast(str, href), propstats, status) + ## TODO: there is currently quite some overlapping with the protocol.xml_parsers + ## we should refactor def _find_objects_and_props(self) -> dict[str, dict[str, _Element]]: """Internal implementation of find_objects_and_props without deprecation warning.""" self.objects: dict[str, dict[str, _Element]] = {} @@ -353,7 +339,11 @@ def _expand_simple_prop( return values[0] ## TODO: word "expand" does not feel quite right. - def expand_simple_props( + ## TODO: I'm considering to deprecate this in v4 + def expand_simple_props(self, *largs, **kwargs) -> dict[str, dict[str, str]]: + return self._expand_simple_props(*largs, **kwargs) + + def _expand_simple_props( self, props: Iterable[BaseElement] | None = None, multi_value_props: Iterable[Any] | None = None, diff --git a/caldav/search.py b/caldav/search.py index a7105a4a..7ace8c1e 100644 --- a/caldav/search.py +++ b/caldav/search.py @@ -22,7 +22,7 @@ ) from .collection import Calendar as AsyncCalendar -TypesFactory = TypesFactory() +_icalendar_types = TypesFactory() # Re-export for backward compatibility diff --git a/docs/design/V3_CODE_REVIEW.md b/docs/design/V3_CODE_REVIEW.md index 13f5aaee..4f136972 100644 --- a/docs/design/V3_CODE_REVIEW.md +++ b/docs/design/V3_CODE_REVIEW.md @@ -5,6 +5,15 @@ **Branch:** v3.0-dev (119 commits since v2.2.6) **Scope:** All changes between tags v2.2.6 and HEAD +> **Status update (commit ff6db30):** All "Must Fix" and "Should Fix" items from §8 have been +> resolved. Remaining open items are v3.1+ and v4.0 work. +> +> **Status update (commit f7c51c9):** Recommendation 11 largely addressed — sync/async +> duplication reduced by ~177 lines across five files (see §5 for details). +> +> **Status update (commit ad042bb):** §1.1 issues 2, 3, 4 fixed (dead code deleted, +> CalendarInfo collision resolved, response/xml_parsers duplication reduced). + ## Executive Summary The v3.0 release is a major architectural refactoring introducing Sans-I/O separation, full async support, and comprehensive API modernization -- all while maintaining backward compatibility with v2.x. The scope is large: 159 files changed, ~25,900 lines added, ~4,500 removed. @@ -12,11 +21,11 @@ The v3.0 release is a major architectural refactoring introducing Sans-I/O separ **Overall assessment:** The architecture is well-designed and the codebase is in good shape for an alpha release. The Sans-I/O protocol layer is clean and testable. However, there are several bugs that should be fixed before a stable release, significant code duplication between sync/async paths (~650 lines), and some test coverage gaps. **Key findings:** -- 3 bugs that will cause runtime errors -- 1 security concern (UUID1 leaks MAC address in calendar UIDs) -- ~650 lines of sync/async duplication across domain objects +- ~~3 bugs that will cause runtime errors~~ **fixed** +- ~~1 security concern (UUID1 leaks MAC address in calendar UIDs)~~ **fixed** +- ~~650 lines of sync/async duplication across domain objects~~ reduced to ~475 lines (commit f7c51c9) - Test coverage gaps in discovery module and sync client unit tests -- `breakpoint()` left in production code +- ~~`breakpoint()` left in production code~~ **fixed** --- @@ -28,20 +37,20 @@ The protocol layer separates XML construction/parsing from I/O. This is the stro | File | Lines | Rating | Purpose | |------|-------|--------|---------| -| `types.py` | 243 | 9/10 | Frozen dataclasses: DAVRequest, DAVResponse, PropfindResult, CalendarQueryResult | -| `xml_builders.py` | 428 | 7/10 | Pure functions building XML for PROPFIND, calendar-query, MKCALENDAR, etc. | -| `xml_parsers.py` | 455 | 5/10 | Parse XML responses into typed results -- has a runtime bug | +| `types.py` | ~~243~~ 221 | 9/10 | Frozen dataclasses: DAVRequest, DAVResponse, PropfindResult, CalendarQueryResult | +| `xml_builders.py` | ~~428~~ 346 | 7/10 | Pure functions building XML for PROPFIND, calendar-query, MKCALENDAR, etc. | +| `xml_parsers.py` | ~~455~~ 467 | 6/10 | Parse XML responses into typed results | | `__init__.py` | 46 | 8/10 | Clean re-exports | **Issues found:** -1. **BUG: NameError in `xml_parsers.py:260`** -- `parse_calendar_multiget_response()` calls `parse_calendar_query_response()` (no leading underscore), but only `_parse_calendar_query_response()` is defined. This will crash at runtime when calendar-multiget responses are parsed. +1. ~~**BUG: NameError in `xml_parsers.py:260`**~~ **FIXED** -- `parse_calendar_multiget_response()` was calling `parse_calendar_query_response()` (missing leading underscore). Fixed by adding the `_` prefix. -2. **Dead code in `xml_builders.py`** -- `_to_utc_date_string()` (line 401), `_build_freebusy_query_body()` (line 189), and `_build_mkcol_body()` (line 200) have zero callers. +2. ~~**Dead code in `xml_builders.py`**~~ **FIXED** -- Deleted `_build_freebusy_query_body`, `_build_mkcol_body`, and `_to_utc_date_string` (the last was a never-wired duplicate of the real `_to_utc_date_string` in `elements/cdav.py`). -3. **`CalendarInfo` name collision** -- `protocol/types.py:149` and `operations/calendarset_ops.py:24` define different dataclasses named `CalendarInfo` with different fields. Both are exported from their respective `__init__.py`. +3. ~~**`CalendarInfo` name collision**~~ **FIXED** -- Deleted `CalendarInfo` from `protocol/types.py` (it was never instantiated); the canonical one in `operations/calendarset_ops.py` is unaffected. -4. **Heavy duplication with `response.py`** -- Multistatus stripping, status validation, response element parsing, and the Confluence `%2540` workaround are duplicated nearly verbatim between `xml_parsers.py` and `response.py`. +4. ~~**Heavy duplication with `response.py`**~~ **PARTIALLY FIXED** -- Extracted `_normalize_href()` (Confluence %2540 fix + absolute-URL-to-path) into `xml_parsers.py`; `response.py._strip_to_multistatus` and `validate_status` now delegate to their `xml_parsers` counterparts. The propstat-loop duplication (`_find_objects_and_props` vs `_extract_properties`) is intentionally left: converging them requires changing the return type of the public `expand_simple_props()` API. ### 1.2 Operations Layer (`caldav/operations/`) @@ -59,13 +68,13 @@ Pure functions for CalDAV business logic. Well-structured but has some issues. **Issues found:** -1. **SECURITY: UUID1 leaks MAC address (`calendarobject_ops.py:55`)** -- `uuid.uuid1()` embeds the host MAC address into generated calendar UIDs. Since calendar events are routinely shared, this is a privacy leak. Should use `uuid.uuid4()`. +1. ~~**SECURITY: UUID1 leaks MAC address (`calendarobject_ops.py:55`)**~~ **FIXED** -- Replaced `uuid.uuid1()` with `uuid.uuid4()`. 2. **`search_ops.py` mutates inputs (line 381-389)** -- `_build_search_xml_query` calls `setattr(searcher, flag, True)` on the passed-in searcher object. This violates the "pure functions" contract and causes side effects in callers. -3. **MD5 in FIPS environments (`calendar_ops.py:132`)** -- `hashlib.md5()` without `usedforsecurity=False` will fail in FIPS-mode environments. Used for fake sync tokens, not for security. +3. ~~**MD5 in FIPS environments (`calendar_ops.py:132`)**~~ **FIXED** -- Added `usedforsecurity=False` to both `calendar_ops.py` and `collection.py`. -4. **Duplicate URL quoting** -- `quote(uid.replace("/", "%2F"))` pattern appears at both lines 62 and 138 in `calendarobject_ops.py`. +4. ~~**Duplicate URL quoting**~~ **FIXED** -- Extracted `_quote_uid()` helper in `calendarobject_ops.py`; `_generate_url`, `_find_id_and_path`, and `calendarobjectresource.CalendarObjectResource._generate_url` all now call it. `quote` import removed from `calendarobjectresource.py`. ### 1.3 Data State Management (`caldav/datastate.py`) @@ -73,11 +82,11 @@ Pure functions for CalDAV business logic. Well-structured but has some issues. ### 1.4 Response Handling (`caldav/response.py`) -**Rating: 6/10** -- `BaseDAVResponse` provides shared XML parsing for sync/async clients, but has significant duplication with the protocol layer and thread-unsafe mutable state. +**Rating: 7/10** -- `BaseDAVResponse` provides shared XML parsing for sync/async clients. The main duplication with the protocol layer has been reduced; remaining open item is thread-unsafe mutable state. **Issues found:** -1. **Unguarded index access (line 169)** -- `_strip_to_multistatus` accesses `tree[0]` without checking `len(tree) > 0` when `tree.tag == "xml"`. The equivalent code in `xml_parsers.py:279` does guard this. +1. ~~**Unguarded index access (line 169)**~~ **FIXED** -- Added `len(tree) > 0` guard before `tree[0]` access in `_strip_to_multistatus`, matching the equivalent guard in `xml_parsers.py`. 2. **Thread-unsafe** -- `self.objects`, `self.results`, `self._responses` are mutable instance state set during parsing. If a response object is shared between threads, results could be corrupt. @@ -100,17 +109,20 @@ Pure functions for CalDAV business logic. Well-structured but has some issues. **Issues found:** -1. **BUG: `HTTPBearerAuth` incompatible with httpx (line 862)** -- `HTTPBearerAuth` is a requests/niquests `AuthBase` subclass that implements `__call__(self, r)`. httpx uses a different auth protocol (`httpx.Auth` with `auth_flow`). Bearer auth will fail at runtime on the httpx code path. +1. ~~**BUG: `HTTPBearerAuth` incompatible with httpx**~~ **FIXED** -- Added `_HttpxBearerAuth(httpx.Auth)` class with `auth_flow` generator inside the httpx import block. `build_auth_object` now uses it on the httpx path and falls back to `HTTPBearerAuth` for niquests. + +2. ~~**BUG: Missing `url.unauth()` call**~~ **FIXED** -- Added `self.url = self.url.unauth()` after credentials are extracted from the URL, matching the sync client behaviour. -2. **BUG: Missing `url.unauth()` call** -- The sync client strips credentials from URLs at `davclient.py:342`, but the async client never does. Credentials embedded in URLs could leak in log messages. +3. ~~**Rate-limit handling asymmetry**~~ **FIXED** -- `AsyncDAVClient.request()` now has `rate_limit_time_slept: float = 0`, the same adaptive backoff loop as the sync client (each retry adds half the already-slept time), and the same features-based auto-detect of `rate_limit_handle`. Retries recursively via `self.request()` instead of calling `_async_request()` directly. -3. **`_auto_url` blocks the event loop** -- RFC6764 discovery performs synchronous DNS lookups and HTTP requests inside `AsyncDAVClient.__init__()`, which is called from `async get_davclient()`. This blocks the event loop. +4. **`_auto_url` blocks the event loop** -- RFC6764 discovery performs synchronous DNS lookups and HTTP requests inside `AsyncDAVClient.__init__()`, which is called from `async get_davclient()`. This blocks the event loop. -4. **Sync import dependency** -- `async_davclient.py:198` imports from `caldav.davclient`, pulling the sync HTTP stack into async contexts. `_auto_url` should live in `caldav/config.py` or `caldav/discovery.py`. +5. **Sync import dependency** -- `async_davclient.py:198` imports from `caldav.davclient`, pulling the sync HTTP stack into async contexts. `_auto_url` should live in `caldav/config.py` or `caldav/discovery.py`. -5. **Password encoding asymmetry** -- Sync client encodes password to bytes eagerly (`davclient.py:329`), async does not. This creates different code paths for auth building. +6. **Password encoding asymmetry** -- Sync client encodes password to bytes eagerly (`davclient.py:329`), async does not. This creates different code paths for auth building. + +7. **Response parsing boilerplate** -- The pattern `if response.status in (200, 207) and response._raw: ...` is repeated in ~5 methods. Should be a helper. -6. **Response parsing boilerplate** -- The pattern `if response.status in (200, 207) and response._raw: ...` is repeated in ~5 methods. Should be a helper. ### 2.3 Sync Client (`caldav/davclient.py`) @@ -118,13 +130,15 @@ Pure functions for CalDAV business logic. Well-structured but has some issues. **Issues found:** -1. **Bare `except:` clauses (lines 367, 679)** -- Catches `SystemExit` and `KeyboardInterrupt`. Should be `except Exception:`. +1. ~~**Bare `except:` clauses (lines 367, 679)**~~ **FIXED** -- Narrowed to `except TypeError` for the test teardown fallback (the expected exception when `teardown()` takes an argument), and `except Exception` for `check_dav_support` (which intentionally catches anything from principal lookup and falls back to root URL). + +2. ~~**Rate-limit auto-detect always enabled**~~ **FIXED** -- `is_supported('rate-limit', dict)` returns `{'enable': False}` (a truthy non-empty dict) for any unconfigured client; the `if rate_limit:` check was incorrectly auto-enabling rate limiting for everyone. Changed to `if rate_limit and rate_limit.get('enable'):`. Also fixed a `TypeError` crash when `rate_limit_max_sleep is None` and the `>` comparison was attempted. -2. **`NotImplementedError` for auth failures (line 996)** -- When no supported auth scheme is found, `NotImplementedError` is raised. Should be `AuthorizationError`. +3. ~~**`NotImplementedError` for auth failures (line 996)**~~ **ALREADY FIXED** -- Code already calls `self._raise_authorization_error()` which raises `AuthorizationError`. Issue was stale at time of review. -3. **Type annotation gaps** -- Multiple `headers: Mapping[str, str] = None` parameters where `None` is not in the union type. +4. **Type annotation gaps** -- Multiple `headers: Mapping[str, str] = None` parameters where `None` is not in the union type. -4. **`propfind` API divergence** -- Sync version (line 754) takes `props=None` which can be either XML string or property list. Async version (line 512) has separate `body` and `props` parameters. +5. **`propfind` API divergence** -- Sync version (line 754) takes `props=None` which can be either XML string or property list. Async version (line 512) has separate `body` and `props` parameters. ### 2.4 Lazy Imports (`caldav/__init__.py`) @@ -153,7 +167,7 @@ Minor: `WWW-Authenticate` parsing (line 31) splits on commas, which fails for he **Rating: 7/10** -- Solid dual-mode foundation. **Issues:** -1. **Production-unsafe assert (`line 99`)** -- `assert " " not in str(self.url)` will be silently removed with `-O`. Also passes when `url=None`. +1. ~~**Production-unsafe assert (`line 99`)**~~ **FIXED** -- Replaced `assert " " not in str(self.url)` with an explicit `ValueError` (also fixed the `url=None` case that the assert silently passed). 2. **Return type lies for async** -- Methods like `get_property()`, `get_properties()`, `delete()` return coroutines when used with async clients, but annotations say `str | None`, `dict`, `None`. 3. **`set_properties` regression** -- Changed from per-property status checking to HTTP status-only checking, losing ability to detect partial PROPPATCH failures. @@ -162,18 +176,18 @@ Minor: `WWW-Authenticate` parsing (line 31) splits on commas, which fails for he **Rating: 6/10** -- Functional but at 2,054 lines is the largest file and could benefit from extraction. **Issues:** -1. **Missing deprecation warnings** -- `calendars()`, `events()`, `todos()` etc. have docstring notes but no `warnings.warn()` calls (unlike `date_search` and `davobject.name` which do emit). -2. **`_generate_fake_sync_token` uses MD5 (line 1655)** -- Same FIPS concern as calendar_ops.py. -3. **`Principal._async_get_property` overrides parent (line 352)** with incompatible implementation. +1. **Missing deprecation warnings** -- `calendars()`, `events()`, `todos()` etc. have docstring notes but no `warnings.warn()` calls (unlike `date_search` and `davobject.name` which do emit). *Comment:* This is intentional. Those are "core features" that was suddenly changed between v2 and v3 - we want people to upgrade to v3 without getting a lot of deprecation warnings thrown in their face. Those warnings will be added in v4. +2. ~~**`_generate_fake_sync_token` uses MD5 (line 1655)**~~ **FIXED** -- Added `usedforsecurity=False`. +3. ~~**`Principal._async_get_property` overrides parent (line 352)** with incompatible implementation.~~ **FIXED** -- Deleted the override; parent `DAVObject._async_get_property` handles it correctly. ### 3.3 CalendarObjectResource (`caldav/calendarobjectresource.py`) **Rating: 7/10** -- Good DataState integration but large (1,919 lines). **Issues:** -1. **BUG: `_set_deprecated_vobject_instance` (line 1248)** -- Calls `self._get_vobject_instance(inst)` but `_get_vobject_instance` takes no arguments. Will raise `TypeError` when invoked via `event.instance = some_vobject`. -2. **`id` setter is a no-op (line 123)** -- `id` passed to constructor is silently ignored. -3. **`_async_load` missing multiget fallback** -- Sync `load()` has `load_by_multiget()` fallback, async does not. +1. ~~**BUG: `_set_deprecated_vobject_instance` (line 1248)**~~ **FIXED** -- Was calling `_get_vobject_instance(inst)` (getter, wrong number of arguments); fixed to call `_set_vobject_instance(inst)`. +2. **`id` setter is a no-op (line 123)** -- Intentional design: UID lives in the iCalendar data, not as a separate field. The setter exists to satisfy the parent class `__init__` without side effects; removing it would cause silent writes that don't take effect. Not a bug. +3. ~~**`_async_load` missing multiget fallback**~~ **FIXED** -- Added `_async_multiget()` to `Calendar` and `_async_load_by_multiget()` to `CalendarObjectResource`; `_async_load` now has the same fallback chain as sync `load()`. 4. **Dual data model risk** -- Old `_data`/`_vobject_instance`/`_icalendar_instance` coexist with new `_state`. Manual sync at lines 1206, 1279 could desynchronize. ### 3.4 Search (`caldav/search.py`) @@ -203,38 +217,46 @@ Minor: `WWW-Authenticate` parsing (line 31) splits on commas, which fails for he **Rating: 7/10** -- Comprehensive server database. **Issues:** -1. **`breakpoint()` in production code (line 443)** -- `FeatureSet._default()` has `breakpoint()` in the `else` branch. Will pause execution in production. -2. **Deprecated `incompatibility_description` dict still present (lines 660-778)** -- Marked "TO BE REMOVED" with 30+ entries. +1. ~~**`breakpoint()` in production code (line 443)**~~ **FIXED** -- Replaced with `raise ValueError(f"Unknown feature type: {feature_type!r}")`. +2. **`incompatibility_description` dict still present (lines 660-778)** -- Marked "TO BE REMOVED" but still referenced by `test_caldav.py` (lines 722, 727, 754). Cannot be deleted until the test code is updated. 3. **`# fmt: off` for entire 1,366-line file** -- Should scope it to just the dict definitions. --- ## 5. Code Duplication Analysis -### 5.1 Sync/Async Client Duplication (~70 lines) +### 5.1 Sync/Async Client Duplication (~30 lines remaining) -| Code Section | davclient.py | async_davclient.py | Similarity | -|---|---|---|---| -| `search_principals` | 376-435 | 1107-1168 | ~95% (copy-paste + await) | -| `_get_calendar_home_set` | 548-568 | 974-994 | ~95% | -| `get_events` | 570-597 | 996-1023 | ~95% | -| `get_todos` | 599-613 | 1025-1039 | ~95% | -| `propfind` response parsing | 280-320 | 750-790 | ~90% | -| Auth type extraction | 180-210 | 420-450 | ~100% | -| Factory functions | 1015-1078 | 1312-1431 | ~80% | +| Code Section | davclient.py | async_davclient.py | Similarity | Status | +|---|---|---|---|---| +| `search_principals` | 376-435 | 1107-1168 | ~95% (copy-paste + await) | **FIXED** — build/parse hoisted to `BaseDAVClient` | +| `_get_calendar_home_set` | 548-568 | 974-994 | ~95% | open | +| `get_events` | 570-597 | 996-1023 | ~95% | **FIXED** — hoisted to `BaseDAVClient` | +| `get_todos` | 599-613 | 1025-1039 | ~95% | **FIXED** — hoisted to `BaseDAVClient` | +| `propfind` response parsing | 280-320 | 750-790 | ~90% | open | +| Auth type extraction | 180-210 | 420-450 | ~100% | already in `BaseDAVClient` | +| Factory functions | 1015-1078 | 1312-1431 | ~80% | open | + +### 5.2 Domain Object Async/Sync Duplication (~445 lines remaining) -### 5.2 Domain Object Async/Sync Duplication (~580 lines) +| File | Duplicated pairs | Approx. lines | Notes | +|---|---|---|---| +| davobject.py | 6 method pairs | ~~180~~ ~140 | `_build_xml_body` / `_build_propfind_root` extracted; `_query`/`_async_query` and `set_properties`/`_async_set_properties` shortened; `_query_properties`/`_async_query_properties` each now 1 line | +| collection.py | 8 method pairs | ~~250~~ ~215 | `CalendarSet.get_calendars` sync migrated to protocol layer; shared `_calendars_from_results` eliminates 30+ lines of duplication | +| calendarobjectresource.py | 4 method pairs | ~100 | open | +| search.py | 2 method pairs | ~50 | open | -| File | Duplicated pairs | Approx. lines | -|---|---|---| -| davobject.py | 6 method pairs | ~180 | -| collection.py | 8 method pairs | ~250 | -| calendarobjectresource.py | 4 method pairs | ~100 | -| search.py | 2 method pairs | ~50 | +### 5.3 Protocol/Response Duplication (partially resolved) -### 5.3 Protocol/Response Duplication +`response.py` and `protocol/xml_parsers.py` originally shared five pieces of nearly identical logic. Three have been consolidated: -`response.py` and `protocol/xml_parsers.py` share five pieces of nearly identical logic (multistatus stripping, status validation, response element parsing, `%2540` workaround, propstat loops). +| Piece | Status | +|---|---| +| Multistatus stripping | **FIXED** — `response._strip_to_multistatus` now delegates to `xml_parsers._strip_to_multistatus` | +| Status validation | **FIXED** — `response.validate_status` now delegates to `xml_parsers._validate_status` | +| `%2540` Confluence workaround + absolute-URL-to-path | **FIXED** — extracted as `xml_parsers._normalize_href`; called from both `_parse_response_element` and `response._parse_response` | +| Response element parsing (`_parse_response` vs `_parse_response_element`) | open — too much divergence in error handling to merge without risk | +| Propstat loops (`_find_objects_and_props` vs `_extract_properties`) | open — `_find_objects_and_props` must return raw `_Element` objects for `expand_simple_props()`; merging requires a public API change | --- @@ -244,10 +266,10 @@ Minor: `WWW-Authenticate` parsing (line 31) splits on commas, which fails for he |---|---|---|---| | `caldav/protocol/` | Excellent | 9/10 | Pure unit tests in test_protocol.py | | `caldav/operations/` | Excellent | 9/10 | 6 dedicated test files | -| `caldav/async_davclient.py` | Good | 8/10 | test_async_davclient.py (821 lines) | +| `caldav/async_davclient.py` | Good | 8/10 | test_async_davclient.py; adaptive rate-limit tests added | | `caldav/datastate.py` | Good | 7/10 | Covered through calendarobject tests | | `caldav/search.py` | Good | 7/10 | test_search.py + integration tests | -| `caldav/davclient.py` | Poor | 4/10 | Only integration tests, no unit tests | +| `caldav/davclient.py` | Low | 5/10 | Rate-limit unit tests added to test_caldav_unit.py; rest still integration-only | | `caldav/collection.py` | Moderate | 6/10 | Integration tests cover most paths | | `caldav/discovery.py` | None | 0/10 | Zero dedicated tests | | `caldav/config.py` | Poor | 3/10 | Module docstring says "test coverage is poor" | @@ -263,55 +285,57 @@ Minor: `WWW-Authenticate` parsing (line 31) splits on commas, which fails for he ## 7. Bugs Summary (Ordered by Severity) -| # | Severity | Location | Description | -|---|---|---|---| -| 1 | HIGH | `xml_parsers.py:260` | NameError: calls `parse_calendar_query_response` (missing underscore) | -| 2 | HIGH | `async_davclient.py:862` | `HTTPBearerAuth` incompatible with httpx -- bearer auth broken on httpx path | -| 3 | MEDIUM | `calendarobjectresource.py:1248` | `_set_deprecated_vobject_instance` passes arg to no-arg getter | -| 4 | MEDIUM | `compatibility_hints.py:443` | `breakpoint()` in production code path | -| 5 | MEDIUM | `async_davclient.py` | Missing `url.unauth()` -- credential leak in logs | -| 6 | MEDIUM | `calendarobject_ops.py:55` | UUID1 leaks MAC address in calendar UIDs | -| 7 | LOW | `davclient.py:367,679` | Bare `except:` catches SystemExit/KeyboardInterrupt | -| 8 | LOW | `response.py:169` | Unguarded `tree[0]` access | -| 9 | LOW | `davobject.py:99` | Production-unsafe `assert` for URL validation | +| # | Severity | Location | Description | Status | +|---|---|---|---|---| +| 1 | HIGH | `xml_parsers.py:260` | NameError: calls `parse_calendar_query_response` (missing underscore) | **FIXED** | +| 2 | HIGH | `async_davclient.py` | `HTTPBearerAuth` incompatible with httpx -- bearer auth broken on httpx path | **FIXED** | +| 3 | MEDIUM | `calendarobjectresource.py:1248` | `_set_deprecated_vobject_instance` passes arg to no-arg getter | **FIXED** | +| 4 | MEDIUM | `compatibility_hints.py:443` | `breakpoint()` in production code path | **FIXED** | +| 5 | MEDIUM | `async_davclient.py` | Missing `url.unauth()` -- credential leak in logs | **FIXED** | +| 6 | MEDIUM | `calendarobject_ops.py:55` | UUID1 leaks MAC address in calendar UIDs | **FIXED** | +| 7 | LOW | `davclient.py:367,679` | Bare `except:` catches SystemExit/KeyboardInterrupt | **FIXED** | +| 8 | LOW | `response.py:169` | Unguarded `tree[0]` access | **FIXED** | +| 9 | LOW | `davobject.py:99` | Production-unsafe `assert` for URL validation | **FIXED** | --- ## 8. Recommendations -### For v3.0 Stable Release (Must Fix) +### For v3.0 Stable Release (Must Fix) -- ALL DONE ✓ -1. Fix the NameError in `xml_parsers.py:260` (add underscore) -2. Remove `breakpoint()` from `compatibility_hints.py:443` -3. Fix `_set_deprecated_vobject_instance` to call setter not getter -4. Replace `uuid.uuid1()` with `uuid.uuid4()` in `calendarobject_ops.py:55` -5. Fix bare `except:` to `except Exception:` in `davclient.py` -6. Add `url.unauth()` call to `AsyncDAVClient.__init__` +1. ~~Fix the NameError in `xml_parsers.py:260` (add underscore)~~ +2. ~~Remove `breakpoint()` from `compatibility_hints.py:443`~~ +3. ~~Fix `_set_deprecated_vobject_instance` to call setter not getter~~ +4. ~~Replace `uuid.uuid1()` with `uuid.uuid4()` in `calendarobject_ops.py:55`~~ +5. ~~Fix bare `except:` to narrower exception types in `davclient.py`~~ +6. ~~Add `url.unauth()` call to `AsyncDAVClient.__init__`~~ -### For v3.0 Stable Release (Should Fix) +### For v3.0 Stable Release (Should Fix) -- ALL DONE ✓ -7. Fix `HTTPBearerAuth` for httpx path (or document httpx+bearer as unsupported) -8. Add guard for `tree[0]` access in `response.py:169` -9. Replace production `assert` with proper validation in `davobject.py:99` -10. Add `usedforsecurity=False` to MD5 calls for FIPS compliance +7. ~~Fix `HTTPBearerAuth` for httpx path~~ +8. ~~Add guard for `tree[0]` access in `response.py:169`~~ +9. ~~Replace production `assert` with proper validation in `davobject.py:99`~~ +10. ~~Add `usedforsecurity=False` to MD5 calls for FIPS compliance~~ ### For v3.1+ -11. Reduce sync/async client duplication (move `search_principals`, `get_events`, `get_todos` to operations layer) -12. Consolidate `response.py` and `protocol/xml_parsers.py` duplication +11. ~~Reduce sync/async client duplication (`search_principals`, `get_events`, `get_todos`)~~ **DONE** — all three hoisted to `BaseDAVClient`; XML build/parse extracted to `_build_principal_search_query` / `_parse_principal_search_response`; `_build_xml_body` / `_build_propfind_root` helpers added to `DAVObject`; `CalendarSet.get_calendars` sync path migrated to protocol layer +12. ~~Consolidate `response.py` and `protocol/xml_parsers.py` duplication~~ **PARTIALLY DONE** — three of five duplicated pieces consolidated (see §5.3); response-element parsing and propstat loops remain 13. Add sync DAVClient unit tests mirroring async test structure 14. Add discovery module tests 15. Add missing `warnings.warn()` to all deprecated methods -16. Remove dead code in `xml_builders.py` -17. Move `_auto_url` from `davclient.py` to shared module +16. ~~Remove dead code in `xml_builders.py`~~ **DONE** — deleted `_build_freebusy_query_body`, `_build_mkcol_body`, `_to_utc_date_string` +17. Move `_auto_url` from `davclient.py` to shared module (also fixes event-loop blocking in async client) 18. Make `search_ops._build_search_xml_query` not mutate its input +19. Fix `NotImplementedError` for auth failures in `davclient.py` -- raise `AuthorizationError` instead +20. Fix `_auto_url` blocking the event loop in `AsyncDAVClient.__init__` ### For v4.0 -19. Address implicit data conversion side effects (issue #613) -20. Consider splitting `collection.py` (2,054 lines) and `calendarobjectresource.py` (1,919 lines) -21. Fix return type annotations for async-capable methods (use `@overload`) -22. Remove `incompatibility_description` dict from compatibility_hints.py +21. Address implicit data conversion side effects (issue #613) +22. Consider splitting `collection.py` (2,054 lines) and `calendarobjectresource.py` (1,919 lines) +23. Fix return type annotations for async-capable methods (use `@overload`) +24. Remove `incompatibility_description` dict from compatibility_hints.py --- @@ -321,13 +345,13 @@ Minor: `WWW-Authenticate` parsing (line 31) splits on commas, which fails for he |------|-------|---------| | `caldav/async_davclient.py` | 1,431 | Async HTTP client | | `caldav/base_client.py` | 480 | Shared client ABC | -| `caldav/response.py` | 390 | Shared response parsing | +| `caldav/response.py` | ~~390~~ 374 | Shared response parsing | | `caldav/datastate.py` | 246 | Data representation state machine | | `caldav/aio.py` | 93 | Async entry point | | `caldav/lib/auth.py` | 69 | Shared auth utilities | -| `caldav/protocol/types.py` | 243 | Request/response dataclasses | -| `caldav/protocol/xml_builders.py` | 428 | XML construction | -| `caldav/protocol/xml_parsers.py` | 455 | XML parsing | +| `caldav/protocol/types.py` | ~~243~~ 221 | Request/response dataclasses | +| `caldav/protocol/xml_builders.py` | ~~428~~ 346 | XML construction | +| `caldav/protocol/xml_parsers.py` | ~~455~~ 467 | XML parsing | | `caldav/operations/base.py` | 189 | Query specifications | | `caldav/operations/search_ops.py` | 445 | Search query building | | `caldav/operations/calendarobject_ops.py` | 531 | Calendar object ops | diff --git a/pyproject.toml b/pyproject.toml index 65949c7b..1cbf742b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ dependencies = [ "niquests", "recurring-ical-events>=2.0.0", "typing_extensions;python_version<'3.11'", - "icalendar>6.0.0", + "icalendar>7.0.0", "icalendar-searcher>=1.0.5,<2", "dnspython", "python-dateutil", diff --git a/tests/caldav_test_servers.yaml.example b/tests/caldav_test_servers.yaml.example index 48fa1ccb..ee9efcf1 100644 --- a/tests/caldav_test_servers.yaml.example +++ b/tests/caldav_test_servers.yaml.example @@ -109,6 +109,14 @@ test-servers: username: ${ZIMBRA_USERNAME:-testuser@zimbra.io} password: ${ZIMBRA_PASSWORD:-testpass} + stalwart: + type: docker + enabled: ${TEST_STALWART:-false} + host: ${STALWART_HOST:-localhost} + port: ${STALWART_PORT:-8809} + username: ${STALWART_USERNAME:-testuser} + password: ${STALWART_PASSWORD:-testpass} + # ========================================================================= # External/private servers (your own CalDAV server) # ========================================================================= diff --git a/tests/docker-test-servers/nextcloud/setup_nextcloud.sh b/tests/docker-test-servers/nextcloud/setup_nextcloud.sh index dc30579e..6e2f0a8e 100755 --- a/tests/docker-test-servers/nextcloud/setup_nextcloud.sh +++ b/tests/docker-test-servers/nextcloud/setup_nextcloud.sh @@ -38,7 +38,7 @@ echo "Enabling contacts app..." docker exec $CONTAINER_NAME php occ app:enable contacts || true echo "Disabling rate limiting for testing..." -docker exec $CONTAINER_NAME php occ config:system:set ratelimit.enabled --value=false --type=boolean || true +#docker exec $CONTAINER_NAME php occ config:system:set ratelimit.enabled --value=false --type=boolean || true docker exec $CONTAINER_NAME php occ app:disable bruteforcesettings || true docker exec $CONTAINER_NAME php occ config:system:set auth.bruteforce.protection.enabled --value=false --type=boolean || true diff --git a/tests/docker-test-servers/stalwart/docker-compose.yml b/tests/docker-test-servers/stalwart/docker-compose.yml new file mode 100644 index 00000000..f6682537 --- /dev/null +++ b/tests/docker-test-servers/stalwart/docker-compose.yml @@ -0,0 +1,18 @@ +services: + stalwart: + image: stalwartlabs/stalwart:latest + container_name: stalwart-test + ports: + - "8809:8080" + environment: + - STALWART_ADMIN_PASSWORD=adminpass + tmpfs: + - /opt/stalwart/data:size=500m + - /opt/stalwart/logs:size=50m + - /opt/stalwart/etc:size=10m + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/"] + interval: 5s + timeout: 3s + retries: 15 + start_period: 25s diff --git a/tests/docker-test-servers/stalwart/setup_stalwart.sh b/tests/docker-test-servers/stalwart/setup_stalwart.sh new file mode 100755 index 00000000..4fddb0d6 --- /dev/null +++ b/tests/docker-test-servers/stalwart/setup_stalwart.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# Setup script for Stalwart test server. +# Creates the test domain and user via the management REST API. +# +# Stalwart requires: +# 1. A domain principal before a user can be created with that email. +# 2. A user principal with a plain-text secret (Stalwart hashes it internally). +# +# CalDAV is served at /dav/cal// over plain HTTP on port 8080. + +set -e + +CONTAINER_NAME="stalwart-test" +HOST_PORT="${STALWART_PORT:-8809}" +ADMIN_USER="admin" +ADMIN_PASSWORD="adminpass" +DOMAIN="example.org" +TEST_USER="${STALWART_USERNAME:-testuser}" +TEST_PASSWORD="${STALWART_PASSWORD:-testpass}" +API_BASE="http://localhost:${HOST_PORT}/api" + +api_post() { + local endpoint="$1" + local body="$2" + curl -s -X POST "${API_BASE}${endpoint}" \ + -H "Content-Type: application/json" \ + -H "Accept: application/json" \ + -u "${ADMIN_USER}:${ADMIN_PASSWORD}" \ + -d "${body}" +} + +echo "Waiting for Stalwart HTTP endpoint to be ready..." +max_attempts=60 +for i in $(seq 1 $max_attempts); do + if curl -s -o /dev/null -w "%{http_code}" "http://localhost:${HOST_PORT}/" 2>/dev/null | grep -q "200"; then + echo "Stalwart is ready" + break + fi + if [ $i -eq $max_attempts ]; then + echo "Stalwart did not start in time" + docker logs "$CONTAINER_NAME" 2>&1 | tail -20 + exit 1 + fi + echo -n "." + sleep 1 +done + +echo "" +echo "Creating domain '${DOMAIN}'..." +RESULT=$(api_post "/principal" "{\"type\": \"domain\", \"name\": \"${DOMAIN}\"}") +if echo "$RESULT" | grep -q '"error"'; then + if echo "$RESULT" | grep -q '"fieldAlreadyExists"'; then + echo "Domain already exists (OK)" + else + echo "Warning: domain creation returned: $RESULT" + fi +fi + +echo "Creating test user '${TEST_USER}'..." +RESULT=$(api_post "/principal" "{ + \"type\": \"individual\", + \"name\": \"${TEST_USER}\", + \"secrets\": [\"${TEST_PASSWORD}\"], + \"emails\": [\"${TEST_USER}@${DOMAIN}\"], + \"roles\": [\"user\"] +}") +if echo "$RESULT" | grep -q '"error"'; then + if echo "$RESULT" | grep -q '"fieldAlreadyExists"'; then + echo "User already exists (OK)" + else + echo "Error creating user: $RESULT" + exit 1 + fi +else + echo "User created: $RESULT" +fi + +echo "" +echo "Verifying CalDAV access..." +max_caldav_attempts=15 +for i in $(seq 1 $max_caldav_attempts); do + RESPONSE=$(curl -s -X PROPFIND \ + -H "Depth: 0" \ + -u "${TEST_USER}:${TEST_PASSWORD}" \ + "http://localhost:${HOST_PORT}/dav/cal/${TEST_USER}/" 2>/dev/null) + if echo "$RESPONSE" | grep -qi "multistatus\|collection"; then + echo "CalDAV is accessible" + break + fi + if [ $i -eq $max_caldav_attempts ]; then + echo "Warning: CalDAV access test failed after ${max_caldav_attempts} attempts" + echo "Response: $RESPONSE" + echo "Continuing anyway..." + break + fi + echo -n "." + sleep 2 +done + +echo "" +echo "Stalwart setup complete!" +echo "" +echo "Credentials:" +echo " Admin: ${ADMIN_USER} / ${ADMIN_PASSWORD} (web UI: http://localhost:${HOST_PORT})" +echo " Test user: ${TEST_USER} / ${TEST_PASSWORD}" +echo " CalDAV URL: http://localhost:${HOST_PORT}/dav/cal/${TEST_USER}/" diff --git a/tests/docker-test-servers/stalwart/start.sh b/tests/docker-test-servers/stalwart/start.sh new file mode 100755 index 00000000..b9e7d0fc --- /dev/null +++ b/tests/docker-test-servers/stalwart/start.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Quick start script for Stalwart test server +# +# Usage: ./start.sh + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +echo "Creating and starting Stalwart container..." +docker-compose up -d + +echo "Running setup (waits for HTTP readiness, creates domain and test user)..." +bash "$SCRIPT_DIR/setup_stalwart.sh" + +echo "" +echo "Run tests from project root:" +echo " cd ../../.." +echo " TEST_STALWART=true pytest" +echo "" +echo "To stop Stalwart: ./stop.sh" +echo "To view logs: docker-compose logs -f stalwart" diff --git a/tests/docker-test-servers/stalwart/stop.sh b/tests/docker-test-servers/stalwart/stop.sh new file mode 100755 index 00000000..05760c1f --- /dev/null +++ b/tests/docker-test-servers/stalwart/stop.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Stop script for Stalwart test server + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +echo "Stopping Stalwart and removing volumes..." +docker-compose down -v + +echo "Stalwart stopped and volumes removed" diff --git a/tests/test_async_davclient.py b/tests/test_async_davclient.py index 8e3f2995..3ad5a3b1 100644 --- a/tests/test_async_davclient.py +++ b/tests/test_async_davclient.py @@ -935,3 +935,37 @@ async def test_rate_limit_max_sleep_zero_raises(self): ) with pytest.raises(error.RateLimitError): await client.request("/") + + @pytest.mark.asyncio + async def test_rate_limit_adaptive_sleep_increases_on_repeated_retries(self): + """On repeated 429s the sleep grows: first sleep uses Retry-After, second adds half of already-slept.""" + ok_response = self._make_response(200) + client = AsyncDAVClient(url="http://cal.example.com/", rate_limit_handle=True) + client.session.request = AsyncMock( + side_effect=[ + self._make_response(429, {"Retry-After": "4"}), + self._make_response(429, {"Retry-After": "4"}), + ok_response, + ] + ) + with patch("caldav.async_davclient.asyncio.sleep", new_callable=AsyncMock) as mock_sleep: + response = await client.request("/") + assert mock_sleep.call_count == 2 + sleeps = [c.args[0] for c in mock_sleep.call_args_list] + assert sleeps[0] == 4.0 + assert sleeps[1] == 6.0 # 4 + 4/2 + assert response.status == 200 + assert client.session.request.call_count == 3 + + @pytest.mark.asyncio + async def test_rate_limit_max_sleep_stops_adaptive_retries(self): + """When accumulated sleep exceeds rate_limit_max_sleep, retrying stops.""" + client = AsyncDAVClient( + url="http://cal.example.com/", rate_limit_handle=True, rate_limit_max_sleep=5 + ) + client.session.request = AsyncMock( + return_value=self._make_response(429, {"Retry-After": "4"}) + ) + with patch("caldav.async_davclient.asyncio.sleep", new_callable=AsyncMock): + with pytest.raises(error.RateLimitError): + await client.request("/") diff --git a/tests/test_caldav_unit.py b/tests/test_caldav_unit.py index 4f2f0322..2321789f 100755 --- a/tests/test_caldav_unit.py +++ b/tests/test_caldav_unit.py @@ -2019,3 +2019,34 @@ def test_rate_limit_max_sleep_zero_raises(self, mocked): ) with pytest.raises(error.RateLimitError): client.request("/") + + @mock.patch("caldav.davclient.requests.Session.request") + def test_rate_limit_adaptive_sleep_increases_on_repeated_retries(self, mocked): + """On repeated 429s the sleep grows: first sleep uses Retry-After, second adds half of already-slept.""" + ok_response = mock.MagicMock() + ok_response.status_code = 200 + ok_response.headers = {} + mocked.side_effect = [ + self._make_response(429, {"Retry-After": "4"}), + self._make_response(429, {"Retry-After": "4"}), + ok_response, + ] + client = DAVClient(url="http://cal.example.com/", rate_limit_handle=True) + with mock.patch("caldav.davclient.time.sleep") as mock_sleep: + response = client.request("/") + assert mock_sleep.call_count == 2 + assert mock_sleep.call_args_list[0] == mock.call(4.0) + assert mock_sleep.call_args_list[1] == mock.call(6.0) # 4 + 4/2 + assert response.status == 200 + assert mocked.call_count == 3 + + @mock.patch("caldav.davclient.requests.Session.request") + def test_rate_limit_max_sleep_stops_adaptive_retries(self, mocked): + """When accumulated sleep exceeds rate_limit_max_sleep, retrying stops.""" + mocked.return_value = self._make_response(429, {"Retry-After": "4"}) + client = DAVClient( + url="http://cal.example.com/", rate_limit_handle=True, rate_limit_max_sleep=5 + ) + with mock.patch("caldav.davclient.time.sleep"): + with pytest.raises(error.RateLimitError): + client.request("/") diff --git a/tests/test_servers/docker.py b/tests/test_servers/docker.py index 645dfd0c..ce6f7860 100644 --- a/tests/test_servers/docker.py +++ b/tests/test_servers/docker.py @@ -2,7 +2,7 @@ Docker-based test server implementations. This module provides test server implementations for servers that run -in Docker containers: Baikal, Nextcloud, Cyrus, SOGo, Bedework, DAViCal, Davis, CCS, and Zimbra. +in Docker containers: Baikal, Nextcloud, Cyrus, SOGo, Bedework, DAViCal, Davis, CCS, Zimbra, and Stalwart. """ import os @@ -349,6 +349,35 @@ def is_accessible(self) -> bool: return False +class StalwartTestServer(DockerTestServer): + """ + Stalwart all-in-one mail & collaboration server in Docker. + + Stalwart added CalDAV/CardDAV support in 2024/2025. Uses plain HTTP on + port 8080 for both the admin interface and CalDAV access. Calendar home + for a user is at /dav/cal//. + """ + + name = "Stalwart" + + def __init__(self, config: dict[str, Any] | None = None) -> None: + config = config or {} + config.setdefault("host", os.environ.get("STALWART_HOST", "localhost")) + config.setdefault("port", int(os.environ.get("STALWART_PORT", "8809"))) + config.setdefault("username", os.environ.get("STALWART_USERNAME", "testuser")) + config.setdefault("password", os.environ.get("STALWART_PASSWORD", "testpass")) + if "features" not in config: + config["features"] = compatibility_hints.stalwart.copy() + super().__init__(config) + + def _default_port(self) -> int: + return 8809 + + @property + def url(self) -> str: + return f"http://{self.host}:{self.port}/dav/cal/{self.username}/" + + # Register server classes register_server_class("baikal", BaikalTestServer) register_server_class("nextcloud", NextcloudTestServer) @@ -359,3 +388,4 @@ def is_accessible(self) -> bool: register_server_class("davis", DavisTestServer) register_server_class("ccs", CCSTestServer) register_server_class("zimbra", ZimbraTestServer) +register_server_class("stalwart", StalwartTestServer) diff --git a/tests/test_servers/embedded.py b/tests/test_servers/embedded.py index 2ef62fbf..1adf9d3a 100644 --- a/tests/test_servers/embedded.py +++ b/tests/test_servers/embedded.py @@ -215,7 +215,13 @@ def start(self) -> None: return try: - from xandikos.web import XandikosApp, XandikosBackend + from xandikos.web import XandikosApp + + try: + # xandikos master renamed XandikosBackend to SingleUserFilesystemBackend + from xandikos.web import SingleUserFilesystemBackend as XandikosBackend + except ImportError: + from xandikos.web import XandikosBackend # type: ignore[no-redef] except ImportError as e: raise RuntimeError("Xandikos is not installed") from e @@ -228,7 +234,7 @@ def start(self) -> None: self.serverdir.__enter__() # Create backend and configure principal (following conf.py pattern) - backend = XandikosBackend(path=self.serverdir.name) + backend = XandikosBackend(self.serverdir.name) backend._mark_as_principal(f"/{self.username}/") backend.create_principal(f"/{self.username}/", create_defaults=True)