Merge pull request #1415 from datajoint/fix/1410-filepath-directory-support

feat: support directory references in <filepath@store> (fixes #1410)

Author: dimitri-yatsenko

src/datajoint/builtin_codecs/filepath.py

@@ -145,17 +145,22 @@ def encode(self, value: Any, *, key: dict | None = None, store_name: str | None
         if not backend.exists(path):
             raise FileNotFoundError(f"File not found in store '{store_name or 'default'}': {path}")
 
-        # Get file info
-        try:
-            size = backend.size(path)
-        except Exception:
-            size = None
+        # Detect whether the path is a directory or a file
+        is_dir = backend.isdir(path)
+
+        # Get file size (not applicable for directories)
+        size = None
+        if not is_dir:
+            try:
+                size = backend.size(path)
+            except (FileNotFoundError, OSError):
+                pass
 
         return {
             "path": path,
             "store": store_name,
             "size": size,
-            "is_dir": False,
+            "is_dir": is_dir,
             "timestamp": datetime.now(timezone.utc).isoformat(),
         }
 

src/datajoint/objectref.py

@@ -379,7 +379,8 @@ def _verify_folder(self) -> bool:
         manifest_path = f"{self.path}.manifest.json"
 
         if not self._backend.exists(manifest_path):
-            raise IntegrityError(f"Manifest file missing: {manifest_path}")
+            # Directory was stored without a manifest — treat as unverified but valid
+            return True
 
         # Read manifest
         manifest_data = self._backend.get_buffer(manifest_path)

src/datajoint/storage.py

@@ -555,7 +555,7 @@ def get_buffer(self, remote_path: str | PurePosixPath) -> bytes:
 
     def exists(self, remote_path: str | PurePosixPath) -> bool:
         """
-        Check if a file exists in storage.
+        Check if a path (file or directory) exists in storage.
 
         Parameters
         ----------
@@ -565,15 +565,28 @@ def exists(self, remote_path: str | PurePosixPath) -> bool:
         Returns
         -------
         bool
-            True if file exists.
+            True if the path exists.
         """
         full_path = self._full_path(remote_path)
         logger.debug(f"exists: {self.protocol}:{full_path}")
+        return self.fs.exists(full_path)
 
-        if self.protocol == "file":
-            return Path(full_path).is_file()
-        else:
-            return self.fs.exists(full_path)
+    def isdir(self, remote_path: str | PurePosixPath) -> bool:
+        """
+        Check if a path refers to a directory in storage.
+
+        Parameters
+        ----------
+        remote_path : str or PurePosixPath
+            Path in storage.
+
+        Returns
+        -------
+        bool
+            True if the path is a directory.
+        """
+        full_path = self._full_path(remote_path)
+        return self.fs.isdir(full_path)
 
     def remove(self, remote_path: str | PurePosixPath) -> None:
         """

tests/unit/test_codecs.py

@@ -548,6 +548,7 @@ def test_filepath_allows_user_paths(self):
             with patch("datajoint.hash_registry.get_store_backend") as mock_get_backend:
                 mock_backend = MagicMock()
                 mock_backend.exists.return_value = True
+                mock_backend.isdir.return_value = False
                 mock_backend.size.return_value = 1024
                 mock_get_backend.return_value = mock_backend
 
@@ -636,6 +637,7 @@ def test_filepath_enforces_filepath_prefix(self):
             with patch("datajoint.hash_registry.get_store_backend") as mock_get_backend:
                 mock_backend = MagicMock()
                 mock_backend.exists.return_value = True
+                mock_backend.isdir.return_value = False
                 mock_backend.size.return_value = 3072
                 mock_get_backend.return_value = mock_backend